evocation/historical/quine.asm

;;; QUINE
;;;
;;; This file is formatted to be read at 80-columns or wider.
;;;
;;; There's some tabular information, but diagrams have been avoided, in an
;;; attempt to make this manageable in screen readers. Feedback welcome.
;;;
;;; First the hex
;;; Second the spark
;;; Third the words
;;; Then tie the knot
;;;   [draft]


;;;;;;;;;;;;;;;;;;;;;
;;; What is this? ;;;
;;;;;;;;;;;;;;;;;;;;;
;;;
;;;   This file was the original version of Evocation, used before the
;;; self-hosting version existed. It's unlikely to ever be needed again, but
;;; Irenes are proud of it and have kept it here for historical, sentimental
;;; reasons.
;;;
;;;   Those reasons are also why it's called "quine". It is not a Quine, in
;;; the sense that it is not a program that outputs its own source. Since it
;;; is of purely historical interest, this misnomer does not matter.


;;;;;;;;;;;;;;;;;;;;;
;;; Workflow tips ;;;
;;;;;;;;;;;;;;;;;;;;;
;;;
;;; This is not fully self-hosting; it is based on flatassembler[1]. A minimal
;;; command to build and run it is:
;;;
;;; $ fasmg quine.asm quine && chmod 755 quine && ./quine; echo $?
;;;
;;; A workflow you may wish to use for debugging is:
;;;
;;; $ rm quine2; fasmg quine.asm quine && chmod 755 quine && ./quine > quine2; echo "exit code:" $?; echo; hexdump -C quine; echo; hexdump -C quine2; echo; cmp -l quine quine2 ; echo cmp: $?
;;;
;;; The reason this removes the old one first is that otherwise, there's a
;;; risk the error message will be scrolled off the top of the screen and
;;; you'll see stale output and not realize.
;;;
;;; You may also wish to do:
;;;
;;; $ objdump --disassemble quine
;;; $ ZydisDisasm -64 quine
;;;
;;; This relies on GNU binutils, and on zydis, respectively.
;;;
;;; [1] https://flatassembler.net/
;;;
;;;
;;; gdb
;;; ---
;;;
;;; You can run gdb on it if you want; there's no symbols, but if you are
;;; familiar with the hex it should be readable. Keep a hexdump of the program
;;; handy to look up what addresses are.
;;;
;;; If you want to see a routine implemented in assembly, look at the hexdump
;;; of the overall file, find it by looking at the ASCII names, skip past the
;;; codeword, and do ie
;;;
;;; (gdb) disassemble/r 0x80007c0,+32
;;;
;;; If you want to see the value stack, you can do
;;;
;;; (gdb) x/16xg $rsp
;;;
;;; The same will work with $rbp for the control stack, and don't forget that
;;; the "instruction pointer" is rsi. To see all the registers, do
;;;
;;; (gdb) info registers


;;;;;;;;;;;;;;;;;;;;;;;;;
;;; Assembly language ;;;
;;;;;;;;;;;;;;;;;;;;;;;;;
;;;
;;;   Before doing any actual code, we define macros for writing x86-64
;;; assembly language. This is built from scratch, relying only on
;;; flatassembler's built-in semantics. No include files of any kind are used
;;; for it.

macro pad: bytes
  if bytes > 0
    db 0x00
    pad (bytes - 1)
  end if
end macro

macro align: bytes
  if bytes > 0
    if $ mod bytes <> 0
      db 0x00
      align bytes
    end if
  end if
end macro

; The way these are all spelled out like this is slightly ridiculous, there
; must be a better way.
macro rex.0
  db 0x40
end macro
macro rex.w
  db 0x48
end macro
macro rex.r
  db 0x44
end macro
macro rex.x
  db 0x42
end macro
macro rex.b
  db 0x41
end macro
macro rex.wr
  db 0x4C
end macro
macro rex.wx
  db 0x4A
end macro
macro rex.wb
  db 0x49
end macro
macro rex.rx
  db 0x46
end macro
macro rex.rb
  db 0x45
end macro
macro rex.xb
  db 0x43
end macro
macro rex.wrx
  db 0x4E
end macro
macro rex.wrb
  db 0x4D
end macro
macro rex.wxb
  db 0x4B
end macro
macro rex.rxb
  db 0x47
end macro
macro rex.wrxb
  db 0x4F
end macro

macro modrm mod, reg, rm
  assert mod >= 0 & mod < 4
  assert reg >= 0 & reg < 8
  assert rm >= 0 & rm < 8
  db (mod shl 6) OR (reg shl 3) OR rm
end macro

macro sib scale, index, base
  assert scale >= 0 & scale < 4
  assert index >= 0 & index < 8
  assert base >= 0 & index < 8
  db (scale shl 6) OR (index shl 3) OR base
end macro

macro opcodereg opcode, reg
  assert opcode >= 0 & opcode < 256 & opcode AND 7 = 0
  assert reg >= 0 & reg < 8
  db opcode OR reg
end macro

macro opcodecc opcode, cc
  assert opcode >= 0 & opcode < 256 & opcode AND 15 = 0
  assert cc >= 0 & cc < 16
  db opcode OR cc
end macro

macro scalefield sfield, scale
  if 1 = scale
    sfield = 0
  else if 2 = scale
    sfield = 1
  else if 4 = scale
    sfield = 2
  else if 8 = scale
    sfield = 3
  else
    assert 0
  end if
end macro

; Yep, there sure is a lot of duplication in these. This is based on Intel's
; documented mnemonics...
;
; "Above" and "below" are for unsigned comparisons. "Greater" and "less" are
; for signed comparisons.
;
; This is documented on the individual opcode pages, and also in B.1.4.7.
macro conditioncode cc, condition
  match =above, condition
    cc = 0x07
  else match =above_equal, condition
    cc = 0x03
  else match =below, condition
    cc = 0x02
  else match =below_equal, condition
    cc = 0x06
  else match =carry, condition
    cc = 0x02
  else match =equal, condition
    cc = 0x04
  else match =greater, condition
    cc = 0x0F
  else match =greater_equal, condition
    cc = 0x0D
  else match =less, condition
    cc = 0x0C
  else match =less_equal, condition
    cc = 0x0E
  else match =not_above, condition
    cc = 0x06
  else match =not_above_equal, condition
    cc = 0x02
  else match =not_below, condition
    cc = 0x03
  else match =not_below_equal, condition
    cc = 0x07
  else match =not_carry, condition
    cc = 0x03
  else match =not_equal, condition
    cc = 0x05
  else match =not_greater, condition
    cc = 0x0E
  else match =not_greater_equal, condition
    cc = 0x0C
  else match =not_less, condition
    cc = 0x0D
  else match =not_less_equal, condition
    cc = 0x0F
  else match =not_overflow, condition
    cc = 0x01
  else match =not_parity, condition
    cc = 0x0B
  else match =not_sign, condition
    cc = 0x09
  else match =not_zero, condition
    cc = 0x05
  else match =overflow, condition
    cc = 0x00
  else match =parity, condition
    cc = 0x0A
  else match =parity_even, condition
    cc = 0x0A
  else match =parity_odd, condition
    cc = 0x0B
  else match =sign, condition
    cc = 0x08
  else match =zero, condition
    cc = 0x04
  else
    assert 0
  end match
end macro


;;; On registers
;;; ------------
;;;
;;;   The x86 architecture has been around a while, it has been through
;;; several transitions from smaller word sizes to larger ones. Therefore it
;;; has different names for the "same" registers, depending on how much of
;;; them you're using.
;;;
;;; TODO there's more to write here

macro bytereg result, register
  match =al?, register
    result = 0
  else match =cl?, register
    result = 1
  else match =dl?, register
    result = 2
  else match =bl?, register
    result = 3
  else match =ah?, register
    result = 4
  else match =ch?, register
    result = 5
  else match =dh?, register
    result = 6
  else match =bh?, register
    result = 7
  else
    assert 0
  end match
end macro

macro wordreg result, register
  match =ax?, register
    result = 0
  else match =cx?, register
    result = 1
  else match =dx?, register
    result = 2
  else match =bx?, register
    result = 3
  else match =sp?, register
    result = 4
  else match =bp?, register
    result = 5
  else match =si?, register
    result = 6
  else match =di?, register
    result = 7
  else
    assert 0
  end match
end macro

macro dwordreg result, register
  match =eax?, register
    result = 0
  else match =ecx?, register
    result = 1
  else match =edx?, register
    result = 2
  else match =ebx?, register
    result = 3
  else match =esp?, register
    result = 4
  else match =ebp?, register
    result = 5
  else match =esi?, register
    result = 6
  else match =edi?, register
    result = 7
  else
    assert 0
  end match
end macro

macro qwordreg result, register
  match =rax?, register
    result = 0
  else match =rcx?, register
    result = 1
  else match =rdx?, register
    result = 2
  else match =rbx?, register
    result = 3
  else match =rsp?, register
    result = 4
  else match =rbp?, register
    result = 5
  else match =rsi?, register
    result = 6
  else match =rdi?, register
    result = 7
  else
    assert 0
  end match
end macro

macro owordreg result, register
  match =r8?, register
    result = 0
  else match =r9?, register
    result = 1
  else match =r10?, register
    result = 2
  else match =r11?, register
    result = 3
  else match =r12?, register
    result = 4
  else match =r13?, register
    result = 5
  else match =r14?, register
    result = 6
  else match =r15?, register
    result = 7
  else
    assert 0
  end match
end macro


;;; Instructions
;;; ------------


macro mov.dreg.dimm target, source
  dwordreg treg, target
  opcodereg 0xB8, treg
  dd source
end macro


macro mov.qreg.dimm target, source
  qwordreg treg, target
  rex.w
  db 0xC7
  modrm 3, 0, treg
  dd source
end macro


macro mov.qreg.qimm target, source
  qwordreg treg, target
  rex.w
  opcodereg 0xB8, treg
  dq source
end macro


; Notice the use of REX.B here; this instruction puts the register number in
; the opcode field, so it uses Table 3-1.
macro mov.oreg.qimm target, source
  owordreg treg, target
  rex.wb
  opcodereg 0xB8, treg
  dq source
end macro


macro mov.qreg.qreg target, source
  qwordreg treg, target
  qwordreg sreg, source
  rex.w
  db 0x89
  modrm 3, sreg, treg
end macro


; Take a 64-bit source register, treat it as an address and look up the 64-bit
; value it points to, store that into a 64-bit target register.
;
; For a source of rbp, the only modes available also have displacement; we
; disallow that. Use mov.qreg.disp8.qreg for that case, and set a displacement
; of 0.
;
; In understanding this, pay close attention to the Op/En column in the opcode
; table. The "RM" variant means the ModRM byte's R/M field (the third one)
; is the source, while its reg field (the middle one) is the target. This is
; what we want, because the R/M field is the one that gets indirection applied
; to it. Opcode 0x8B with an REX.W prefix is the all-64-bit RM variant.
; [Intel] volume 2B, chapter 4, section 4-3, "MOV".
;
; For the indirection modes, don't be confused by the many similar tables.
; 64-bit mode is encoded the same as 32-bit mode except for adding a REX.W
; prefix, as per 2.2.1.1, so you want table 2-2 to understand the ModRM byte.
; The presence or absence of an SIB byte is determined by where in that table
; we fall, and we aren't using a mode that has one. [Intel] volume 2A,
; chapter 2, section 2-1.5, table 2-2.
;
; We disallow rsp as a source because that's the mode that would want an SIB.
macro mov.qreg.indirect.qreg target, source
  match =rbp, source
    assert 0
  end match
  qwordreg sreg, source
  qwordreg treg, target
  rex.w
  db 0x8B
  modrm 0, treg, sreg
  match =rsp, source
    ; R/M = rsp is the SIB case
    sib 0, 4, sreg
      ; no scaling, no indexing, source as base
  end match
end macro


macro mov.qreg.disp8.qreg target, offset, source
  qwordreg sreg, source
  qwordreg treg, target
  rex.w
  db 0x8B
  modrm 1, treg, sreg
  match =rsp, source
    ; R/M = rsp is the SIB case
    sib 0, 4, sreg
      ; no scaling, no indexing, source as base
  end match
  db offset
end macro


; Take a 64-bit source register, store its value into the address pointed to
; by a 64-bit target register.
;
; For a target of rbp, the only modes available also have displacement; we
; disallow that. Use mov.disp8.qreg.qreg for that case, and set a displacement
; of 0.
;
; In understanding this, pay close attention to the Op/En column in the opcode
; table. The "MR" variant means the ModRM byte's reg field (the middle one)
; is the source, while its R/M field (the third one) is the target. This is
; what we want, because the R/M field is the one that gets indirection applied
; to it. Opcode 0x89 with an REX.W prefix is the all-64-bit MR variant.
; [Intel] volume 2B, chapter 4, section 4-3, "MOV".
;
; For the indirection modes, don't be confused by the many similar tables.
; 64-bit mode is encoded the same as 32-bit mode except for adding a REX.W
; prefix, as per 2.2.1.1, so you want table 2-2 to understand the ModRM byte.
; The presence or absence of an SIB byte is determined by where in that table
; we fall, and we aren't using a mode that has one. [Intel] volume 2A,
; chapter 2, section 2-1.5, table 2-2.
;
; We disallow rsp as a target because that's the mode that would want an SIB.
; When you look at other addressing modes, be aware that the special treatment
; is for whichever register is specified in the R/M field. Sometimes that's
; the source, and sometimes it's the target, depending on the opcode.
macro mov.indirect.qreg.qreg target, source
  match =rbp, target
    assert 0
  end match
  qwordreg sreg, source
  qwordreg treg, target
  rex.w
  db 0x89
  modrm 0, sreg, treg
  match =rsp, target
    ; R/M = rsp is the SIB case
    sib 0, 4, treg
      ; no scaling, no indexing, target as base
  end match
end macro


; 8-bit source register
macro mov.indirect.qreg.breg target, source
  match =rsp, target
    assert 0
    ; The SIB case.
  else match =rbp, target
    assert 0
    ; An unrelated addressing mode.
  else
    bytereg sreg, source
    qwordreg treg, target
    db 0x88
    modrm 0, sreg, treg
  end match
end macro

macro mov.breg.indirect.qreg target, source
  match =rsp, source
    assert 0
    ; The SIB case.
  else match =rbp, source
    assert 0
    ; An unrelated addressing mode.
  else
    qwordreg sreg, source
    bytereg treg, target
    db 0x8A
    modrm 0, treg, sreg
  end match
end macro

; We use the operand-size prefix to specify 16-bit. No REX.W. Table 3-4.
macro mov.indirect.qreg.wreg target, source
  match =rsp, target
    assert 0
    ; The SIB case.
  else match =rbp, target
    assert 0
    ; An unrelated addressing mode.
  else
    wordreg sreg, source
    qwordreg treg, target
    db 0x66
    db 0x89
    modrm 0, sreg, treg
  end match
end macro

; We use the operand-size prefix to specify 16-bit. No REX.W. Table 3-4.
macro mov.wreg.indirect.qreg target, source
  match =rsp, source
    assert 0
    ; The SIB case.
  else match =rbp, source
    assert 0
    ; An unrelated addressing mode.
  else
    qwordreg sreg, source
    wordreg treg, target
    db 0x66
    db 0x8B
    modrm 0, treg, sreg
  end match
end macro

; It defaults to 32-bit, no prefix needed, also no REX.W. Table 3-4.
macro mov.indirect.qreg.dreg target, source
  match =rsp, target
    assert 0
    ; The SIB case.
  else match =rbp, target
    assert 0
    ; An unrelated addressing mode.
  else
    dwordreg sreg, source
    qwordreg treg, target
    db 0x89
    modrm 0, sreg, treg
  end match
end macro

; It defaults to 32-bit, no prefix needed, also no REX.W. Table 3-4.
macro mov.dreg.indirect.qreg target, source
  match =rsp, source
    assert 0
    ; The SIB case.
  else match =rbp, source
    assert 0
    ; An unrelated addressing mode.
  else
    qwordreg sreg, source
    dwordreg treg, target
    db 0x8B
    modrm 0, treg, sreg
  end match
end macro


macro mov.qreg.indexed.qreg target, source, index, scale
  match =rbp, source
    assert 0
    ; This is divided into some subcases we don't wish to deal with yet.
  else match =rsp, index
    assert 0
    ; This is the case where it's not actually indexed after all.
  else
    qwordreg treg, target
    qwordreg sreg, source
    qwordreg ireg, index
    scalefield sfield, scale
    rex.w
    db 0x8B
    modrm 0, treg, 4
    sib sfield, ireg, sreg
  end match
end macro


macro mov.indexed.qreg.qreg target, index, scale, source
  match =rbp, source
    assert 0
    ; This is divided into some subcases we don't wish to deal with yet.
  else match =rsp, index
    assert 0
    ; This is the case where it's not actually indexed after all.
  else
    qwordreg treg, target
    qwordreg sreg, source
    qwordreg ireg, index
    scalefield sfield, scale
    rex.w
    db 0x89
    modrm 0, sreg, 4
    sib sfield, ireg, treg
  end match
end macro


; Take a 64-bit source register, store its value into a high 64-bit target
; register (r8-r15).
;
; Notice that there are two ways to add another bit to the register encoding.
; Table 3-1 is about REX.B, but does not apply here, it's for instructions
; that use opcode bits to specify a register, and none of the
; register-to-register MOV variants do that (it's for immediate mode).
;
; Instead, we want the mechanism that uses REX.R as the extra bit, and it
; combines with the reg field of ModRM, as per 2.2.1.2.
;
; Therefore, we want the variant of MOV which puts the target in the reg
; field. That's Op/En "RM", opcode 0x8B with REX.WR.
;
; Mode 3 is direct addressing.
macro mov.oreg.qreg target, source
  owordreg treg, target
  qwordreg sreg, source
  rex.wr
  db 0x8B
  modrm 3, treg, sreg
end macro


; Take a high 64-bit source register (r8-r15), store its value into a 64-bit
; target register.
;
; Notice that there are two ways to add another bit to the register encoding.
; Table 3-1 is about REX.B, but does not apply here, it's for instructions
; that use opcode bits to specify a register, and none of the
; register-to-register MOV variants do that (it's for immediate mode).
;
; Instead, we want the mechanism that uses REX.R as the extra bit, and it
; combines with the reg field of ModRM, as per 2.2.1.2.
;
; Therefore, we want the variant of MOV which puts the source in the reg
; field. That's Op/En "MR", opcode 0x89 with REX.WR.
;
; Mode 3 is direct addressing.
macro mov.qreg.oreg target, source
  qwordreg treg, target
  owordreg sreg, source
  rex.wr
  db 0x89
  modrm 3, sreg, treg
end macro


; This increments a 64-bit register by 1, in place;
macro inc.qreg target
  qwordreg treg, target
  rex.w
  db 0xFF
  modrm 3, 0, treg
    ; The 0 is part of the opcode.
end macro

; This decrements a 64-bit register by 1, in place;
macro dec.qreg target
  qwordreg treg, target
  rex.w
  db 0xFF
  modrm 3, 1, treg
    ; The 1 is part of the opcode.
end macro

; This adds a 64-bit register to another 64-bit register, in place.
macro add.qreg.qreg target, source
  qwordreg treg, target
  qwordreg sreg, source
  rex.w
  db 0x01
  modrm 3, sreg, treg
end macro


macro add.indirect.qreg.qreg target, source
  match =rsp, target
    assert 0
    ; The SIB case.
  else match =rbp, target
    assert 0
    ; An unrelated addressing mode.
  else
    qwordreg treg, target
    qwordreg sreg, source
    rex.w
    db 0x01
    modrm 0, sreg, treg
  end match
end macro


macro add.qreg.indirect.qreg target, source
  match =rsp, source
    assert 0
    ; The SIB case.
  else match =rbp, source
    assert 0
    ; An unrelated addressing mode
  else
    qwordreg treg, target
    qwordreg sreg, source
    rex.w
    db 0x03
    modrm 0, treg, sreg
  end match
end macro


; This adds a signed 8-bit immediate value to a 64-bit register, in place.
;
; Notice the use of 3 as the addressing mode. This says to use the register
; itself. The 0 in the reg field is part of the opcode.
macro add.qreg.bimm target, source
  qwordreg treg, target
  rex.w
  db 0x83
  modrm 3, 0, treg
  db source
end macro


; This adds a signed 32-bit immediate value to a 64-bit register, in place.
;
; Notice the use of 3 as the addressing mode. This says to use the register
; itself. The 0 in the reg field is part of the opcode.
macro add.qreg.dimm target, source
  qwordreg treg, target
  rex.w
  db 0x81
  modrm 3, 0, treg
  dd source
end macro

; This subtracts a 64-bit register from another 64-bit register, in place.
macro sub.qreg.qreg target, source
  qwordreg treg, target
  qwordreg sreg, source
  rex.w
  db 0x2B
  modrm 3, treg, sreg
end macro


macro sub.indirect.qreg.qreg target, source
  match =rsp, target
    ; The SIB case.
    assert 0
  else
    qwordreg treg, target
    qwordreg sreg, source
    rex.w
    db 0x2B
    modrm 0, sreg, treg
  end match
end macro


; This subtracts a signed 8-bit immediate value from a 64-bit register, in
; place.
;
; Notice the use of 3 as the addressing mode. This says to use the register
; itself. The 5 in the reg field is part of the opcode.
macro sub.qreg.bimm target, source
  qwordreg treg, target
  rex.w
  db 0x83
  modrm 3, 5, treg
  db source
end macro

; This subtracts a signed 32-bit immediate value from a 64-bit register, in
; place.
;
; Notice the use of 3 as the addressing mode. This says to use the register
; itself. The 5 in the reg field is part of the opcode.
macro sub.qreg.dimm target, source
  qwordreg treg, target
  rex.w
  db 0x81
  modrm 3, 5, treg
  dd source
end macro

; Add with carry.
macro adc.qreg.bimm target, source
  qwordreg treg, target
  rex.w
  db 0x83
  modrm 3, 2, treg
  db source
end macro

; Subtract with borrow.
macro sbb.qreg.bimm target, source
  qwordreg treg, target
  rex.w
  db 0x83
  modrm 3, 3, treg
  db source
end macro

; This multiplies rax, as 64-bits, with another 64-bit register, in place.
;
; The 4 in the reg field is part of the opcode.
macro mul.rax.qreg source
  qwordreg sreg, source
  rex.w
  db 0xF7
  modrm 3, 4, sreg
end macro

; The official mnemonic for this is "div", but it's divmod: It takes a 128-bit
; dividend formed from concatenating rdx as the high half with rax as the low
; half, and divides it by a 64-bit divisor from a specified register. It
; stores the quotient, truncated towards zero, in rax, and it stores the
; remainder in rdx. This entire process is unsigned.
;
; The 6 in the reg field is part of the opcode.
macro div.rdxrax.qreg source
  qwordreg sreg, source
  rex.w
  db 0xF7
  modrm 3, 6, sreg
end macro

; Same as div, but signed.
;
; The 7 in the reg field is part of the opcode.
macro idiv.rdxrax.qreg source
  qwordreg sreg, source
  rex.w
  db 0xF7
  modrm 3, 7, sreg
end macro


macro and.qreg.qreg target, source
  qwordreg treg, target
  qwordreg sreg, source
  rex.w
  db 0x23
  modrm 3, treg, sreg
end macro

macro and.qreg.bimm target, source
  qwordreg treg, target
  rex.w
  db 0x83
  modrm 3, 4, treg
    ; The 4 is part of the opcode.
  db source
end macro

macro or.qreg.qreg target, source
  qwordreg treg, target
  qwordreg sreg, source
  rex.w
  db 0x0B
  modrm 3, treg, sreg
end macro

macro or.qreg.bimm target, source
  qwordreg treg, target
  rex.w
  db 0x83
  modrm 3, 1, treg
    ; The 4 is part of the opcode.
  db source
end macro

macro xor.qreg.qreg target, source
  qwordreg treg, target
  qwordreg sreg, source
  rex.w
  db 0x33
  modrm 3, treg, sreg
end macro

macro not.qreg target
  qwordreg treg, target
  rex.w
  db 0xF7
  modrm 3, 2, treg
    ; The 2 is part of the opcode.
end macro


; This sets the flags to the same things they'd be set to if subtracting
; right from left.
macro cmp.qreg.qreg left, right
  qwordreg lreg, left
  qwordreg rreg, right
  rex.w
  db 0x3B
  modrm 3, lreg, rreg
end macro

; This sets the flags to the same things they'd be set to if and'ing right
; with left.
macro test.qreg.qreg left, right
  qwordreg lreg, left
  qwordreg rreg, right
  rex.w
  db 0x85
  modrm 3, rreg, lreg
end macro

macro set.breg.cc target, condition
  bytereg treg, target
  conditioncode cc, condition
  db 0x0F
  opcodecc 0x90, cc
  modrm 3, 0, treg
end macro


; Move from an 8-bit immediate value, to a location relative to a 64-bit
; register, with an 8-bit displacement and no indexing.
;
; This uses opcode 0xC6, which has w = 0. Since we run in 64-bit mode, that
; makes the operand size 8 bits, regardless of the current operand-size
; attribute. [Intel] volume 2D, appendix B, section B-1.4.3, table B-6.
macro mov.qreg.disp8.bimm target, offset, source
  qwordreg treg, target
  db 0xC6
  modrm 1, 0, treg
    ; the 0 is part of the opcode
    ; 4 is rsp, but it's a special case
  match =rsp, target
    ; R/M = rsp is the SIB case
    sib 0, 4, treg
      ; no scaling, no indexing, target as base
  end match
  db offset
  db source
end macro

; Move from a 16-bit immediate value, to a location relative to a 64-bit
; register, with an 8-bit displacement and no indexing.
;
; This uses opcode 0xC7, which has w = 1. We run in 64-bit mode, so that gives
; us an operand size of 32 bits by default. [Intel] volume 1, chapter 3,
; section 3-6.1, table 3-4. We want a 16-bit operand, so we use the
; operand-size prefix, 0x66, and we leave REX.W unset.
;
; We need to treat rsp specially because it's the SIB case, per table 2-2.
macro mov.qreg.disp8.wimm target, offset, source
  qwordreg treg, target
  db 0x66
  db 0xC7
  modrm 1, 0, treg
    ; the 0 is part of the opcode
  match =rsp, target
    ; R/M = rsp is the SIB case
    sib 0, 4, treg
      ; no scaling, no indexing, target as base
  end match
  db offset
  dw source
end macro

; Move from a 32-bit immediate value, to a location relative to a 64-bit
; register, with an 8-bit displacement and no indexing.
;
; This uses opcode 0x67, which has w = 1. We run in 64-bit mode, so that gives
; us an operand size of 32 by default. [Intel] volume 2D, section B.1.43,
; table B-6. This is what we want, so we leave it.
macro mov.qreg.disp8.dimm target, offset, source
  qwordreg treg, target
  db 0xC7
  modrm 1, 0, treg
    ; the 0 is part of the opcode
  match =rsp, target
    ; R/M = rsp is the SIB case
    sib 0, 4, treg
      ; no scaling, no indexing, target as base
  end match
  db offset
  dd source
end macro

; Move from a 64-bit register, to a 64-bit location relative to a 64-bit
; register, with an 8-bit displacement and no indexing.
;
; This uses opcode 0x89 with REX.W, so that gives us the reg field as the
; 64-bit source and the R/M field as the 64-bit destination.
;
; We need to treat a target of rsp specially because it's the SIB case per
; table 2-2.
macro mov.disp8.qreg.qreg offset, target, source
  qwordreg sreg, source
  qwordreg treg, target
  rex.w
  db 0x89
  modrm 1, sreg, treg
  match =rsp, source
    ; R/M = rsp is the SIB case
    sib 0, 4, 4
      ; no scaling, no indexing, rsp as base
  end match
  db offset
end macro

; Move from a 64-bit register, to a 64-bit location relative to a 64-bit
; register, with a 32-bit displacement and no indexing.
;
; This uses opcode 0x89 with REX.W, so that gives us the reg field as the
; 64-bit source and the R/M field as the 64-bit destination.
;
; We need to treat a target of rsp specially because it's the SIB case per
; table 2-2.
macro mov.qreg.disp32.qreg target, offset, source
  qwordreg sreg, source
  qwordreg treg, target
  match =rsp, target
    rex.w
    db 0x89
    modrm 2, sreg, treg
      ; treg is rsp by assumption, and R/M = rsp is the SIB case
    sib 0, 4, 4
      ; no scaling, no indexing, rsp as base
    dd offset
  else
    rex.w
    db 0x89
    modrm 2, sreg, treg
    dd offset
  end match
end macro

; Move from a 32-bit immediate value, to a 64-bit location relative to a
; 64-bit register, with an 8-bit displacement and no indexing.
;
; Note that there is no instruction to move a 64-bit immediate to memory.
;
; This uses opcode 0xC7, which has w = 1. We run in 64-bit mode, so that
; gives us an operand size of 32 by default. [Intel] volume 2D,
; section B.1.43, table B-6. We want a 64-bit operand, so we use the REX.W
; prefix, 0x48.
macro mov.qreg.disp8.dimm target, offset, source
  qwordreg treg, target
  match =rsp, target
    rex.w
    db 0xC7
    modrm 1, 0, treg
      ; the 0 is part of the opcode
      ; 4 is rsp, but it's a special case
    sib 0, 4, treg
      ; no scaling, no indexing, rsp as base
    db offset
    dd source
  else
    rex.w
    db 0xC7
    modrm 1, 0, treg
      ; the 0 is part of the opcode
    db offset
    dd source
  end match
end macro

; "Load effective address". Compute a 64-bit address as you would for
;  indexed addressing, with an 8-bit displacement and no indexing, but instead
; of doing anything with the memory, just store the address itself into a
; register.
macro lea.qreg.disp8.qreg target, offset, source
  qwordreg treg, target
  qwordreg sreg, source
  rex.w
  db 0x8D
  modrm 1, treg, sreg
  match =rsp, source
    ; R/M = rsp is the SIB case
    sib 0, 4, sreg
      ; no scaling, no indexing, rsp as base
  end match
  db offset
end macro

macro lea.qreg.disp32.qreg target, offset, source
  qwordreg treg, target
  qwordreg sreg, source
  rex.w
  db 0x8D
  modrm 2, treg, sreg
  match =rsp, source
    ; R/M = rsp is the SIB case
    sib 0, 4, sreg
      ; no scaling, no indexing, rsp as base
  end match
  dd offset
end macro

macro lea.qreg.indexed.qreg target, source, index, scale
  match =rbp, source
    assert 0
    ; This is divided into some subcases we don't wish to deal with yet.
  else match =rsp, index
    assert 0
    ; This is the case where it's not actually indexed after all.
  else
    qwordreg treg, target
    qwordreg sreg, source
    qwordreg ireg, index
    scalefield sfield, scale
    rex.w
    db 0x8D
    modrm 0, treg, 4
    sib sfield, ireg, sreg
  end match
end macro

; Wow, we use ALL the instruction suffixes for this, huh. See [Intel] volume
; 2A, chapter 2, section 2-1, with particular attention to figure 2-1.
macro lea.qreg.disp8.indexed.qreg target, offset, source, index, scale
  match =rbp, source
    assert 0
    ; This is divided into some subcases we don't wish to deal with yet.
  else match =rsp, index
    assert 0
    ; This is the case where it's not actually indexed after all.
  else
    qwordreg treg, target
    qwordreg sreg, source
    qwordreg ireg, index
    scalefield sfield, scale
    rex.w
    db 0x8D
    modrm 1, treg, 4
      ; 1 in the mode field says we want a disp8.
      ; 4 in the R/M field says we want an SIB byte.
    sib sfield, ireg, sreg
    db offset
  end match
end macro

macro mov.breg.breg target, source
  bytereg treg, target
  bytereg sreg, source
  db 0x88
  modrm 3, sreg, treg
end macro

macro mov.breg.bimm target, source
  bytereg treg, target
  db 0xC6
  modrm 3, 0, treg
    ; the 0 is part of the opcode
  db source
end macro

; Clear the DF flag. This makes string instructions increment RSI.
macro cld
  db 0xFC
end macro

; Set the DF flag. This makes string instructions decrement RSI.
macro std
  db 0xFD
end macro

; Load 64 bits from the address in RSI into RAX. Then, increment or decrement
; RSI by 8 bytes, depending on the value of the DF flag.
macro lodsq
  rex.w
  db 0xAD
end macro

macro cmps8
  db 0xA6
end macro

;   [Intel] describes two different styles of mnemonic for the repeated string
; operations. See, their parameters are always rsi and rdi, or the smaller
; versions of those same specific registers. Intel thinks we might want to
; write out "rsi" explicitly, even though the only information it conveys is
; the size. The position we take is that it's better to let that be conveyed
; by the instruction name; otherwise it'd be a point of confusion for new
; readers, who might mistakenly think it's possible to pass it different
; registers.
;
;   With the string instructions, the reader SHOULD be thinking, "Wait...
; where does this get its parameters from?" Writing them in a way that makes
; them appear simpler than they are would be confusing.
macro rep operation
  match =movsb, operation
    db 0xF3 ; rep prefix
    db 0xA4 ; opcode
  else match =movsw, operation
    db 0xF3 ; rep prefix
    db 0x66 ; operand-size prefix
    db 0xA5 ; opcode
  else match =movsd, operation
    db 0xF3 ; rep prefix
    db 0xA5 ; opcode
  else match =movsq, operation
    ; The "rep" instruction can also be thought of as a prefix to other
    ; instructions, though only a few specific ones are allowed. Anyway, it
    ; comes before the REX byte.
    db 0xF3
    ; The rest of this is the same as the encoding of normal, non-repeated
    ; movsq.
    rex.w
    db 0xA5
    ; There's no explicit parameters. String operations are magic.
  else
    assert 0
  end match
end macro

macro repz operation
  match =scasb, operation
    db 0xF3 ; rep prefix
    db 0xAe ; opcode
  else
    assert 0
  end match
end macro

macro repnz operation
  match =scasb, operation
    db 0xF2 ; rep prefix
    db 0xAE ; opcode
  else
    assert 0
  end match
end macro

; Push a 64-bit value from a register onto the stack (the one pointed to by
; rsp). Decrement rsp, then write the value at the new location.
;
; In the corner case where rsp is also the value being pushed, the old value
; is the one used.
;
; There's an alternate encoding of this that uses a ModRM byte, but doing it
; without is more compact, so we do without.
macro push.qreg source
  qwordreg sreg, source
  opcodereg 0x50, sreg
end macro

macro push.bimm source
  db 0x6A
  db source
end macro

; Operand-size prefix makes it 16-bit.
;
; If you're trying to fake pushing a larger size by doing several 16-bit
; pushes, remember to start by pushing the low end and proceed upwards.
; [Intel] volume 1, chapter 9, section 9-2.4, "Memory Data Formats".
macro push.wimm source
  db 0x66
  db 0x68
  dw source
end macro

; There is no 64-bit immediate push. So, can we have a push instruction that
; pushes a 32-bit immediate value? Sort-of, but it's sign-extended to 64 bits,
; so rsp is decremented by 8, not by 4. This is that instruction.
;
; You need to do a really close read of a number of things to understand why.
; The opcode tables in [Intel] in volume 2D, appendix A, section A-3 give it
; the d64 annotation, which per table A-1 in section A-2.5 indicates that the
; operand size is always 64 bits and that there is no corresponding 32-bit
; version. Yet, the actual immediate value is still only 32 bits! Direct your
; attention to the instruction's details page, volume 2B, chapter 4, section
; 4-3, "PUSH". The description section clearly details that the immediate may
; be less than the operand size, which makes sense once you know it, but it
; doesn't explictly call out that the operand size is still 64 bits here.
;
; In general, the size of an immediate doesn't determine operand size, as you
; can read about in detail in [Intel] volume 1, chapter 3, section 3-6.1, with
; particular attention to table 3-4.
;
; Why is this surprising, given that it's consistent with the behavior of
; other instructions? Well, most instructions don't have such obvious
; side-effects. It's easy to not notice the operand size disagreeing with the
; immediate size when you'e only writing to a register, but changing the stack
; in an unexpected way breaks things much more obviously.
;
; Anyway, if you really want to decrement the stack pointer by 32 bits after
; a push, consider pushing a register.
macro push.dimm source
  db 0x68
  dd source
end macro

; Pop a 64-bit value into a register from the stack (the one pointed to by
; rsp). Read the value from the old location, then increment rsp.
;
; In the corner case where rsp is also the destination being written to, the
; read happens from the old location, then the write causes the increment to
; be irrelevant.
;
; There's an alternate encoding of this that uses a ModRM byte, but doing it
; without is more compact, so we do without.
macro pop.qreg target
  qwordreg treg, target
  opcodereg 0x58, treg
end macro


; Do an absolute indirect jump with a 64-bit register operand. That is: given
; a register which holds a pointer, read another address from the pointed-to
; memory and jump to it.
;
; Technically this is a "near" jump in x86 terms, but we just pretend far
; jumps and segments don't exist. They are still a thing in 64-bit mode, we
; just don't use them.
macro jmp.abs.indirect.qreg location
  qwordreg lreg, location
  db 0xFF
  modrm 0, 4, lreg
end macro

; The location is relative to the start of the instruction immediately
; following the jmp.
macro jmp.rel.bimm location
  db 0xEB
  db location
end macro

; There in no 64-bit immediate "near" jump, so we use 32-bit. It's relatve,
; so that's honestly plenty.
;
; The location is relative to the start of the instruction immediately
; following the jmp.
macro jmp.rel.dimm location
  db 0xE9
  dd location
end macro

; The location is relative to the start of the instruction immediately
; following the jmp.
macro jmp.cc.rel.bimm condition, location
  conditioncode cc, condition
  opcodecc 0x70, cc
  db location
end macro

; The location is relative to the start of the instruction immediately
; following the jmp.
macro jmp.cc.rel.dimm condition, location
  conditioncode cc, condition
  db 0x0F
  opcodecc 0x70, cc
  dd location
end macro

; Invoke a system call provided by the kernel. On Linux, the System V ABI
; describes the semantics of such calls (at least, on x86).
macro syscall
  db 0x0F, 0x05
end macro


; Halts the CPU. We can't actually run this in userspace, but the kernel will
; kill our process or the debugger will break, and those are the outcomes we
; actually want.
macro hlt
  db 0xf4
end macro



;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; Executable file format ;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;
;;;   Before we get into the body of the program, we do a lot of ELF-specific
;;; stuff to ensure that our output is in a format Linux knows how to run.
;;;
;;;   First, we set the origin to load at. This is arbitrary, but it can't be
;;; zero. We tell flatassembler about it because it's used in label
;;; calculations; we can reference it as $$ any time we need it in future.
org 0x08000000

;;;
;;;   Second, we output ELF's top-level file header. The only interesting
;;; thing here is the entry pointer.
;;;
elf_header:
  ; * denotes mandatory fields according to breadbox
  db 0x7F, "ELF"                 ; *magic number
  db 2                           ; 64-bit
  db 1                           ; little-endian
  db 1                           ; ELF header format version 1
  db 0                           ; System-V ABI
  db 8 DUP 0                     ; (padding)

  dw 2                           ; *executable
  dw 0x3E                        ; *Intel x86-64
  dd 1                           ; ELF format version

  dq _start                      ; *entry point
  dq program_header - $$         ; *program header offset
  dq 0                           ; section header offset
  dd 0                           ; processor flags
  dw elf_header_size
  dw program_header_entry_size   ; *
  dw 1                           ; *number of program header entries
  dw 0                           ; section header entry size
  dw 0                           ; number of section header entries
  dw 0                           ; section name string table index
; Save a copy of the size of this chunk for our future reference, by comparing
; the current posiion to the label above.
elf_header_size = $ - elf_header

;;;
;;;   Third, immediately after the ELF file header, we output ELF's program
;;; header, which lists the memory regions ("segments") we want to have and
;;; where we want them to come from. We list just a single region, which is
;;; the entire contents of the ELF file from disk.
;;;
;;;   It would be more typical to use this header to ask the loader to give us
;;; separate code and data segments, and perhaps a stack or heap, but this
;;; keeps things simple, and we can create those things for ourselves later.
;;;
;;;    We do have a little stack space available, though we don't explicitily
;;; request any; the kernel allocates it for us as part of exec() so that it
;;; can pass us argc and argv (which we ignore). That stack space will be at a
;;; random address, different every time, because of ASLR; that's a neat
;;; security feature, so we leave it as-is.
;;;
program_header:
  dd 1                           ; *"loadable" segment type
  dd 0x05                        ; *read+execute permission
  dq 0                           ; *offset in file
  dq $$                          ; *virtual address
    ; required, but can be anything, subject to alignment
  dq 0                           ; physical address (ignored)
  dq file_size                   ; *size in file
  dq file_size                   ; *size in memory
  dq 0                           ; segment alignment
                                 ;   for relocation - will we be ASLR'd?
; Save the size of this chunk, as well.
program_header_entry_size = $ - program_header

; Everything after this point is code or data, not headers, so save the start
; of it for use in size calculations later.
code_start:


;;;;;;;;;;;;;;;;;;;;;;;
;;; Execution model ;;;
;;;;;;;;;;;;;;;;;;;;;;;
;;;
;;;   We use Forth-style dual stacks, one for values and one for control. We
;;; use rsp for values, just like C does. We use rbp for the control stack,
;;; which is a special Forth-y stack: These are pointers into the bodies of
;;; Forth words, not return addresses.
;;;
;;;   The choice of rsp and rbp for the stack pointers imitates Jonesforth;
;;; I'm hopeful that it gives us convenient addressing modes, and will report
;;; back about that when I feel that I understand the implications.
;;;
;;;   In Forth, everything is a "word", including mutable variables.
;;; Conceptually, a word is a unit of execution, which may be implemented
;;; either in machine code or as an array of pointer to other words.
;;;
;;;   This polymorphism is implemented by having each word's contents begin
;;; with a "codeword", which is a pointer to machine code that "interprets"
;;; the rest of the contents. In the case of words implemented in machine
;;; code, the codeword points directly to that code, which is normally right
;;; next to it.
;;;
;;;   Variables, to Forth, are simply one more thing that can be executed; the
;;; effect of executing a variable is to push its address onto the value
;;; stack.
;;;
;;;   We adopt this model of words, codewords, and variables-as-words. It's
;;; really nice how it doesn't force anything else on us, not even a heap,
;;; though we do end up using a heap.
;;;
;;;   We specifically implement a version of calling and returning that Forth
;;; calls indirect threaded code: The control stack is a stack of pointers
;;; into the middle of interpreted words. The interpreter snippet, called
;;; docol, implements calling.  Each word is responsible for making sure
;;; returning works properly. Interpreted words accomplish this by ending with
;;; the word "exit", while machine-code words accomplish it by ending with a
;;; verbatim snippet called "next".
;;;
;;;   Conceptually, "next" returns, but more specifically it accomplishes this
;;; by doing the caller's next dispatch for it; thus control never actually
;;; goes back to the caller's interpreter after initial setup. For performance
;;; reasons, "next" is always inlined, so we define it as a macro.
;;;
;;;   The docol routine is just ordinary code, not a macro. It's defined later
;;; in this file, as a label.
;;;
;;;   Notionally, we could consider not having a dictionary, and not giving
;;; our words names. However, it feels silly to stop when we're so close to
;;; being a full Forth, and using names for things solves a bootstrapping
;;; problem related to heap management - see the write-up of _start about how
;;; the heap is created, below. So, we add an additional header before the
;;; codeword for this purpose.
;;;
;;;   The Forth dictionary is usually a linked list of every word that has
;;; ever been defined, with the newest at the head; the names of words are
;;; stored in string fields, often right next to the link pointer. We adopt
;;; this model, with the field sizes and order shown in the quick reference
;;; below. We break with Forth tradition in one way: Rather than having a
;;; length field, we use a null-terminated string. Thus, there's no length
;;; limit on names. This necessitates breaking out the flags (to be explained
;;; later) into their own byte, rather than taking bits from the length field
;;; for them.
;;;
;;;   There's an important performance consideration: Executable words
;;; reference each other by pointers to their respective codewords. However,
;;; dictionary entries reference each other by pointers to their respective
;;; link fields. Traversing from the link field to the codeword is easy,
;;; though it's a non-constant-time operation: Just walk the string. In order
;;; to make Forth words easy to "decompile", it would be nice to also have a
;;; way to traverse backwards. We solve this by making the name field be
;;; null-terminated at both ends. Fun, yeah?
;;;
;;;
;;;
;;;
;;; --------------------------------------------------------------------------
;;;  Quick Reference
;;; --------------------------------------------------------------------------
;;;
;;; The layout of an interpreted word:
;;;
;;; (overall start)
;;;            0x00 - 0x08                     Link (to next-oldest word)
;;;            0x09 - 0x09  HM00000I           Flags
;;;                                                H - hidden
;;;                                                M - metadata
;;;                                                I - immediate
;;;                                                all other bits reserved
;;; (name start)
;;;            0x0a - 0x0a                     Null byte (terminates name)
;;;            0x0b - name-end - 1             Name, as UTF-8
;;;        name-end - name-end                 Null byte (terminates name)
;;; (padding start)
;;;    name-end + 1 - codeword-start - 1       Zero-pad to 8-byte boundary
;;;    (it's possible this will be zero bytes long)
;;; (codeword start)
;;;      ... + 0x00 - ... + 0x08               Codeword (ie. address of docol)
;;;          (8-byte chunks)                   Addresses of other words
;;;                 - ... (end)                Address of "exit" word
;;;
;;; The layout of a machine-code word is different only from the codeword on:
;;;
;;;      ... + 0x00 - ... + 0x08               Addresss of next byte
;;;      ... + 0x08 - ????                     Arbitrary machine code
;;;                 - ... (end)                Inlined implementation of next
;;;
;;; Also, words always start at 8-byte boundaries.
;;;
;;;
;;; REGISTER usage conventions:
;;;
;;; * rsi is the "instruction pointer" for the "interpreter".
;;;     That is, it points to some word-pointer inside an array of
;;;   word-pointers inside the content of the word they're part of. It always
;;;   points to the next word that should be executed, whose execution hasn't
;;;   begun yet.
;;;
;;; * rbp points to the top of the control stack
;;;     These are former values of rsi, to eventually be returned to, from
;;;   successively older callers as you look further up the stack. The stack
;;;   grows downwards in memory. Since values are kept separately, the only
;;;   thing on the control stack is return addresses, one per layer of call.
;;;
;;; * rsp points to the top of the value stack
;;;     The value stack has no specific format, but it grows downwards in
;;;   memory. In particular there's no concept of stack frames, because items
;;;   on the stack don't belong to any particular word; the value stack in
;;;   Forth is in part a mechanism for passing values between words.
;;;
;;; Additionally, immediately after beginning execution of a word:
;;;
;;; * rax points to the address of the codeword being executed
;;;     The value of rax is purely for the callee's benefit, and does not need
;;;   to be preserved.
;;;
;;;   Other registers are purely discretionary, and are not preserved across
;;; calls.
;;;
;;;
;;; FLAG usage:
;;;
;;; * DF should be 0
;;;   We use lodsq extensively and that makes it increment rsi after using it.
;;;
;;; --------------------------------------------------------------------------

;;;
;;; Macro next
;;; ----------
;;;
;;;   Include this inline at the end of a word implemented in machine-code.
;;; Conceptually, it returns. What it actually does is do the next thing the
;;; caller would do, which is call the next word from the caller's array of
;;; word pointers.
;;;
;;; Registers in:
;;;
;;; * rsi points to the address of the word to execute
;;;
;;; Registers out:
;;;
;;; * rax points to the codeword in the contents of the word that was executed
;;; * rsi points to the next word-address after this one
;;;
;;; Flags
;;; * DF = 0 is required
;;;
macro next
  ; Copy the next word's address from *rsi into rax. Increment rsi (as per the
  ; DF flag).
  lodsq

  ; Load the codeword from the word's contents, and jump to the interpreter it
  ; points to.
  jmp.abs.indirect.qreg rax
end macro

;;;
;;; Macro beforenext
;;; ----------------
;;;
;;;   Sometimes we want to transfer control from a word implemented in
;;; machine-code to another word, without coming back after, as if we were
;;; simply jumping to it. This is an innovation of ours; Jonesforth doesn't do
;;; it.
;;;
;;;   This implementation will work regardless of how the receiving word is
;;; implemented. It impersonates the "next" snippet, setting up rax to point
;;; to the codeword then jumping to the interpreter. Since it doesn't change
;;; the control stack or rsi, when the receiving word eventually invokes
;;; "next"; it will pick up in the same place as if this sending word had done
;;; it.
;;;
;;;   Thus, notionally we are doing just this one transfer of control before
;;; eventually getting around to inlining "next". Hence the name.
;;;
macro beforenext target
  ; Do a permanent transfer of control by setting rax and invoking the
  ; codeword. Of course, we could jump to docol ourselves but this will work
  ; regardless of what the receiving codeword is.
  mov.qreg.qimm rax, target
  jmp.abs.indirect.qreg rax
end macro

;;;
;;; Macros pushcontrol
;;;        popcontrol
;;; ------------------
;;;
;;;   Include these inline to push an address onto the control stack, or pop
;;; one off of it. You will recall the control stack is kept in rbp. The
;;; parameter is given in a user-specified register.
;;;
;;;   Jonesforth's analogous macros are called PUSHRSP and POPRSP but I think
;;; that's super confusing, since rsp is also the name of a register, but a
;;; different one. I guess it was less confusing in 32-bit, since esp doesn't
;;; start with an "r". Anyway, this has to be named something that
;;; distinguishes it from Intel's PUSH and POP opcodes, so...
;;;
;;;   "Load effective address" is just a cute way to do arithmetic on a
;;; register, here. To push or pop we decrement or increment rbp by 8. To
;;; actually interact with the space in the stack, we indirect through rbp.
;;;
;;; Registers in and out:
;;;
;;; * rbp points to the top of the control stack.
;;;
macro pushcontrol source
  lea.qreg.disp8.qreg rbp, -8, rbp
  mov.disp8.qreg.qreg 0, rbp, source
end macro

macro popcontrol target
  mov.qreg.disp8.qreg target, 0, rbp
  lea.qreg.disp8.qreg rbp, 8, rbp
end macro

;;;
;;; Routine _start
;;; --------------
;;;
;;;   This is the entry point of the whole program, the very first code we
;;; actually execute. Linkers traditionally call this _start, and on balance
;;; I think it's probably best to keep that name, though I've honestly never
;;; liked it... Anyway, the ELF header points to it and exec() jumps to it.
;;; Also, though it could be anywhere in the code part of the output, in order
;;; to make the hexdump pretty we put it at the start.
;;;
;;;   The kernel gives us most registers zeroed, and rsp pointing to the
;;; command-line stuff (argc, argv, envp), which is at an ASLR'd address with
;;; some stack space allocated for us, despite the fact we didn't request any.
;;; It also gives us all the flags clear except IF, but we don't rely on that.
;;; Lastly, of course, it loads our code segment and sets the instruction
;;; pointer where we asked; we don't need to check what those addresses are,
;;; because they're not randomized.
;;;
;;;   This routine is really only responsible for one-time initialization.
;;;
;;; Registers in:
;;;
;;; * rsp points to the logical top  of the value stack
;;;     The kernel sets this up for us, and we need to save it somewhere so
;;;   Forth can use it.
;;;
;;; Registers out:
;;;
;;; * rsi points within "quit"
;;;     Quit is the word that's Forth's closest equivalent to main().
;;; * rsp points to the top of the value stack
;;;
;;;   Notably, rbp is still uninitialialized after _start.
;;;
;;; Stack in:
;;;
;;; * argc, argv, envp in the usual Unix way
;;;     We ignore them, though.
;;;
;;; Stack out:
;;;
;;; * The value of "heap", as a pointer
;;;     The meaning of this will be explained below.
;;;
;;; Registers within:
;;;
;;; * rdi points to the base the heap was allocated at, once it exists
;;;     This is the same value that "heap" will hold, once we reach a point
;;;   where we have variables. Of course, variables are stored on the heap,
;;;   hence this temporary measure.
;;;
;;;   We also take this opportunity to define soeme memory layout parameters
;;; that this routine will be responsible for doing something with:
;;;
heap_requested_address = 0x0000001000000000 ; (very arbitrary)
heap_size =              0x0000000001000000 ; 16 MiB
control_stack_size =                0x10000 ; 64 KiB

_start:
  cld                                      ; clear the DF flag

  ;;;
  ;;; Prepare the heap.
  ;;;
  ;;;   We could ask for a data segment in the program header, but where's the
  ;;; fun in that? Instead, we call mmap().
  ;;;
  ;;;   If we wanted the kernel to do ASLR for us, passing address zero would
  ;;; cause it to pick somewhere at random, but instead we choose our own
  ;;; location. It's still not guaranteed to be where we ask for, so we still
  ;;; do the work to record where it wound up. We could pass the "fixed" flag
  ;;; and the kernel would trust us, but this gives us more options for
  ;;; interoperating with other runtimes.
  ;;;
  mov.qreg.qimm rax, 9                       ; mmap()
  mov.qreg.qimm rdi, heap_requested_address  ; address (very arbitrary)
  mov.qreg.qimm rsi, heap_size               ; size (one meg)
  mov.qreg.qimm rdx, 0x07                    ; protection (read+write+execute)
  mov.oreg.qimm r10, 0x22                    ; flags (private+anonymous)
  mov.oreg.qimm r8, 0                        ; file descriptor (ignored)
  mov.oreg.qimm r9, 0                        ; offset (ignored)
  syscall

  ;;;
  ;;;   The return value of the system call is in rax, we'll use it in a sec.
  ;;; We need to save this somewhere in case we ever want to munmap() it;
  ;;; there's no widely-used name for it so we have to make one up. S0 and r0
  ;;; are widely-used names for the physical tops (logical bottoms) of the
  ;;; value and control stacks, respectively, and we will eventually set those
  ;;; up as well, so we should keep those names in mind. The control stack
  ;;; lives within the heap, while the value stack is its own segment. This
  ;;; value, though, is the physical bottom of the segment, meaning that it
  ;;; stays the same even as we allocate and deallocate things within it. This
  ;;; is unlike the two stack pointers, so we give it a name that doesn't
  ;;; suggest similarity: "heap".
  ;;;
  ;;;   Once Forth is fully set up, its internal variables will be accessed
  ;;; through variable-words like any other Forth data, including "heap". To
  ;;; get to that point, though, we need to be able to hold onto variable data
  ;;; between now and then. In fact, if we don't have at least one of "heap"
  ;;; and "here" (its counterpart which points to the logical top end), all
  ;;; our efforts to hold onto anything seem a bit doomed.
  ;;;
  ;;;   So, we temporarily dedicate rdi to "heap" - only within this routine -
  ;;; and store everything else in ways that let us find things by reference
  ;;; to it. We choose rdi because it works with the indexing modes we care
  ;;; about, and its name suggests its function.
  ;;;
  ;;;   The strategy Jonesforth uses is not applicable to us; Jonesforth
  ;;; takes advantage of the linker to let its code segment refer to specific,
  ;;; pre-allocated objects in the data segment. We are our own linker.
  ;;; Hence, this approach.
  ;;;
  ;;;   Keying things off "heap" is the fundamental decision, but to make sure
  ;;; our variables are accessible both during early bootstrapping, and later,
  ;;; we also have to be thoughtful about data structures. More on that in a
  ;;; moment.
  ;;;
  mov.qreg.qreg rdi, rax

  ;;;
  ;;;   We also initialize rbp. We could hold off and let "quit" do this, but
  ;;; doing it now is the easiest way to initialize the r0 variable, since
  ;;; there's no instruction that moves a 64-bit immediate to memory.
  ;;;
  ;;;   This is the moment at which we decide where the control stack starts!
  ;;; Fun, right? "Allocation" is just a fancy word for picking where we want
  ;;; something, then being consistent about it - like placing furniture in
  ;;; your home. See below for a little more thought about why here in
  ;;; particular.
  ;;;
  lea.qreg.disp32.qreg rbp, control_stack_size, rdi

  ;;;
  ;;;   Now we save some stuff onto the heap. These are the locations that
  ;;; will eventually be the backing stores of the Forth variables, but we
  ;;; don't create the word headers yet, since there's no requirement that
  ;;; they be next to the backing stores. We'll do that later, once we have
  ;;; word-writing infrastructure in place. For now, we just use their offsets
  ;;; relative to the physical bottom of the heap, which are fixed.
  ;;;
  ;;;   These will be the permanent homes of these values, though we have
  ;;; copies of them elsewhere while we're still in this routine.
  ;;;
  mov.qreg.disp32.qreg rdi, control_stack_size + 0x00, rdi    ; log
  mov.qreg.disp32.qreg rdi, control_stack_size + 0x08, rsp    ; s0
  mov.qreg.disp32.qreg rdi, control_stack_size + 0x10, rbp    ; r0
  mov.qreg.qimm rax, final_word_name
  mov.qreg.disp32.qreg rdi, control_stack_size + 0x18, rax    ; latest
  lea.qreg.disp32.qreg rax, control_stack_size + 0x28, rdi
  mov.qreg.disp32.qreg rdi, control_stack_size + 0x20, rax    ; here
  ;;;
  ;;; * "log" is the physical bottom of the log
  ;;;     The log grows upwards in memory, so this is also the logical
  ;;;   bottom. This comes from the address mmap() just returned to us.
  ;;;     The rest of quine.asm refers to the log as the heap. It's not a
  ;;;   heap, but it used to be called that. The self-hosted version of
  ;;;   Evocation has the fully revised and reconciled copy of all these
  ;;;   comments, it just felt like unnecessary tedium to do that here as
  ;;;   well.
  ;;; * "s0" is the logical bottom of the value stack
  ;;;     The value stack grows downwards in memory, so this is the physical
  ;;;   top of it. This comes from the stack pointer the kernel initialized us
  ;;;   with.
  ;;; * "r0" is the logical bottom of the control stack
  ;;;     The control stack also grows downwards, so this is its pysical top
  ;;;   as well. We allocate this dedicated space within the heap right here,
  ;;;   in this routine, through our choice of where to put things.
  ;;; * "here" is the physical start of the unallocated space in the heap
  ;;;     We allocate heap space from bottom to top, by incrementing this
  ;;;   value. So, it would also be accurate to say that it points immediately
  ;;;   after the physical top of the allocated space. At any rate, the
  ;;;   address it points to is the first one that hasn't been used yet.
  ;;; * "latest" is the address of the most-recently-defined word's header
  ;;;     Defining new words changes this value.
  ;;;
  ;;;   s0 and r0 are mostly used when we want to initialize or reinitialize
  ;;; their respective stacks - that is, discard all their contents at once.
  ;;;
  ;;;   The value of r0 is the same address these variables start at, so
  ;;; you'll want to do a close read of the implementation of pushcontrol
  ;;; and convince yourself that it only ever writes things just below the rbp
  ;;; address it receives, never right on top of it.
  ;;;
  ;;;   As an aside, by the way, please notice that strictly speaking, r0
  ;;; could be a constant... but it isn't known until runtime, so we might as
  ;;; well make it a variable. That will play nicely with any astonishing
  ;;; memory shenanigans someone might wish to do in the future.
  ;;;
  ;;;   Notice also that "here" points immediately after itself. This is just
  ;;; a convenience, making it the last one like that so that the concern is
  ;;; dealt with in a single place and is easy to keep up-to-date with code
  ;;; changes.
  ;;;
  ;;;   A little more detail about why we offset everything by
  ;;; control_stack_size: We're carving out some space at the bottom of the
  ;;; heap - which grows low-to-high - to be the control stack - which grows
  ;;; high-to-low. So the control stack is allocated out of the heap as a
  ;;; fixed-size, one-time thing, and then the variables come immediately
  ;;; after that. We do need to use 32-bit displacement indexing to access
  ;;; them this way, but that's no big deal.
  ;;;
  ;;;   This is perhaps questionable, they should maybe be separate segments
  ;;; created with separate calls to mmap(), but for now we're not worried
  ;;; about overflow so we use the same allocation for both.
  ;;;
  ;;;   We'll come back to these variables a bit later and generate the word
  ;;; headers that point at them, but now we're almost ready to switch to
  ;;; proper threaded-execution, so we finish that setup first...
  ;;;

  ;;;
  ;;;   Push the value of "heap" onto the value stack so that it can be the
  ;;; breadcrumb the threaded code needs to find... the backing store of
  ;;; "heap". Yes, self-reference can be weird like that sometimes. There's
  ;;; nothing stopping "quit" from reading rdi, it just violates the
  ;;; abstraction...
  ;;;
  push.qreg rdi

  ;;;
  ;;;   We are about to set up rsi, we did rbp already, and rsp came to us
  ;;; already set up. That's all that "next" needs, so take it away!
  ;;;
  mov.qreg.qimm rsi, cold_start
  next

;;;
;;;   This isn't really a routine so much as it's an array of words (exactly
;;; one of them), which is what "next" wants rsi to point to. It's only ever
;;; used this one time, so we just put it right here.
;;;

  align 8
cold_start:
  ;;;   Before handing off to us, _start pushed a single value onto the stack,
  ;;; a pointer to the beginning of the heap. Now, we load our entire Forth
  ;;; implementation onto that heap, beginning with the minimal set of words
  ;;; needed to define more words. We do this because we need variables as
  ;;; infrastructure so we can eventually have dynamic definitions.
  ;;;
  ;;;   There's something non-obvious here: words implemented statically as
  ;;; part of the executable image can't contain things that vary at runtime.
  ;;; That means that even if these words tried to implement some sort of
  ;;; dynamic lookup, they would have no way to find the root of whatever
  ;;; dynamic data structure they use. Dynamism needs to be bootstrapped.
  ;;;
  ;;;   In a more traditional C-style program, static code could look up
  ;;; variables based on fixed addresses that are the same on every run.
  ;;; Failing that, we could dedicate a register to it, though that's a
  ;;; considerable expense. We chose not to do either of those things, because
  ;;; we want the versatility that comes with not being picky about our
  ;;; address space: It allows us to contemplate future improvements such as
  ;;; ASLR, or embedding into other processes that impose their own addressing
  ;;; constraints, or even coexisting with multiple versions of ourselves.
  ;;; That choice does mean we have the hard version of this bootstrapping
  ;;; problem, and copying ourselves to the heap is how we solve it.
  ;;;
  ;;;   We do have the heap address right now, though that won't last. In case
  ;;; it's unclear why not: keeping it on the stack would require all future
  ;;; references to walk the stack, and somehow know when they've reached the
  ;;; bottom. The stack is a good place to keep things with clearly delimited
  ;;; lifetimes and visibility, but when we want something to live for our
  ;;; entire program and be easy to find from any code within it, we need to
  ;;; do something else. Anyway, since we have the address, we can use it for
  ;;; the next little bit of setup.
  ;;;
  ;;;   The first few words we define are our variables, which hardcode the
  ;;; addresses they will return - but since we're doing this at runtime,
  ;;; "hardcoding" can reflect where our heap is. This is the fundamental
  ;;; trick that makes the heap usable.
  ;;;
  ;;;   One more thing to notice: We already allocated the backing stores of
  ;;; these variables, and populated their initial values, in _start. The
  ;;; words we're defining return those same addresses for the same backing
  ;;; stores. So, we have continuity: Stuff defined in terms of the
  ;;; variable-words we're defining now will interoperate with the stuff that
  ;;; we define in the "early" way, which includes those very words. Both the
  ;;; early code and the later code are dealing with the same data structures,
  ;;; they're just using a different technique to find them.
  ;;;
  ;;;   This is the only hardcoding we need to do; by building on top of it,
  ;;; we will soon reach a point where the rest of the system can be defined
  ;;; within itself.
  dq early_heap, litstring, "log", early_variable
  dq early_s0, litstring, "s0", early_variable
  dq early_r0, litstring, "r0", early_variable
  dq early_latest, litstring, "latest", early_variable
  dq early_here, litstring, "here", early_variable

  ;;;   Now we define a heap version of docol. Strictly speaking it doesn't
  ;;; need to be among the first words, it only needs to come before the first
  ;;; words implemented in Forth. However, it's conceptually tidy to have it
  ;;; that way, so that's what we do.
  ;;;
  ;;;   Docol also presents a unique challenge, in that it's two snippets of
  ;;; code and one of them needs to refer to the other. When we use docol as
  ;;; the codeword of a word we're defining, we point to a snippet which acts
  ;;; as an interpreter for the word's body. However, when we look up "docol"
  ;;; in the dictionary, what we get is a word that returns the address of the
  ;;; interpreter snippet, effectively acting as a constant.
  ;;;
  ;;;   One way to make this work would be to use a forward-referencing
  ;;; address using the labels system. However, it turns out that only docol
  ;;; and zbranch would benefit from this, and we drastically simplify our
  ;;; code by reworking things so that no forward reference is needed.
  ;;;
  ;;;   What we do is define the interpreter snippet first, allocating space
  ;;; for the machine code directly out of "here", with no word header nor any
  ;;; dictionary entry pointing to it. We keep track of the address we put
  ;;; that at, then we define the constant to point to it.
  ;;;
  ;;;   While it may seem weird to use space that's "outside" of any word,
  ;;; keep in mind that using the heap in creative ways is part of the spirit
  ;;; of Forth. Jonesforth doesn't have this bootstapping problem, but its
  ;;; variables use this same technique of putting the value before the word
  ;;; header to avoid a forward reference. Also, words don't have end
  ;;; delimiters, so who's to say what's inside or outside them?
  dq early_here, fetch, dup
  dq rsi, pack_pushcontrol
  dq lit, 8, rax, add_reg64_imm8
  dq rax, rsi, mov_reg64_reg64
  dq pack_next
  dq lit, 8, packalign
  dq roll3, swap, early_here_store, swap
  ; Now the interpreter snippet is in-place and "here" points after it, so
  ; that future allocation won't step on it. We also still have a copy of its
  ; start address, which we will now pass to early_variable.
  dq litstring, "docol", early_variable
  ; (While it might be tidy to have a separate "early_constant", it would do
  ; the same thing. Late variables and constants will be different because the
  ; real "variable" word will also be responsible for allocating the backing
  ; store, but the only thing early_variable is doing is returning an
  ; address.)

  ;   Now we define a bunch of ordinary Forth words. Since we'll only ever be
  ; calling these copies from within Forth, we're no longer limited by
  ; flatassembler's syntax, so a lot of them have slightly different names
  ; than they've had up to now. This will be noted below.

  dq litstring, "exit", early_create, early_self_codeword, early_here, fetch
  dq rsi, pack_popcontrol, pack_next
  dq lit, 8, packalign, early_here_store

  dq litstring, "swap", early_create, early_self_codeword, early_here, fetch
  dq rax, pop_reg64, rbx, pop_reg64, rax, push_reg64, rbx, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  dq litstring, "drop", early_create, early_self_codeword, early_here, fetch
  dq rax, pop_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "drop2".
  dq litstring, "2drop", early_create, early_self_codeword, early_here, fetch
  dq rax, pop_reg64, rax, pop_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  dq litstring, "roll", early_create, early_self_codeword, early_here, fetch
  dq rcx, pop_reg64
  dq rcx, dec_reg64
  dq rsp, rcx, lit, 8, rbx, mov_reg64_indexed_reg64
  dq rsi, push_reg64
  dq rsp, rcx, lit, 8, rsi, lea_reg64_indexed_reg64
  dq rsp, rcx, lit, 8, lit, 8, rdi, lea_reg64_disp8_indexed_reg64
  dq std
  dq rep_movs64
  dq cld
  dq rsi, pop_reg64
  dq rbx, rsp, mov_indirect_reg64_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; Jonesforth calls this "-roll" and we could do that, but honestly the name
  ; unroll sounds nicer and it's only a single character longer. You might say
  ; it rolls off the tongue better.
  dq litstring, "unroll", early_create, early_self_codeword, early_here, fetch
  dq rcx, pop_reg64
  dq rcx, dec_reg64
  dq rcx, rdx, mov_reg64_reg64
  dq rsp, rbx, mov_reg64_indirect_reg64
  dq rsi, push_reg64
  dq rsp, lit, 16, rsi, lea_reg64_disp8_reg64
  dq rsp, lit, 8, rdi, lea_reg64_disp8_reg64
  dq rep_movs64
  dq rsi, pop_reg64
  dq rbx, rsp, rdx, lit, 8, mov_indexed_reg64_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "roll3".
  dq litstring, "3roll", early_create, early_self_codeword, early_here, fetch
  dq rax, pop_reg64
  dq rbx, pop_reg64
  dq rcx, pop_reg64
  dq rbx, push_reg64
  dq rax, push_reg64
  dq rcx, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "unroll3".
  dq litstring, "3unroll", early_create, early_self_codeword
  dq early_here, fetch
  dq rax, pop_reg64
  dq rbx, pop_reg64
  dq rcx, pop_reg64
  dq rax, push_reg64
  dq rcx, push_reg64
  dq rbx, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  dq litstring, "dup", early_create, early_self_codeword, early_here, fetch
  dq rax, pop_reg64
  dq rax, push_reg64
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "dup2".
  dq litstring, "2dup", early_create, early_self_codeword, early_here, fetch
  dq rax, pop_reg64
  dq rbx, pop_reg64
  dq rbx, push_reg64
  dq rax, push_reg64
  dq rbx, push_reg64
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "add".
  dq litstring, "+", early_create, early_self_codeword, early_here, fetch
  dq rbx, pop_reg64
  dq rax, pop_reg64
  dq rbx, rax, add_reg64_reg64
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "sub".
  dq litstring, "-", early_create, early_self_codeword, early_here, fetch
  dq rbx, pop_reg64
  dq rax, pop_reg64
  dq rbx, rax, sub_reg64_reg64
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "mul".
  dq litstring, "*", early_create, early_self_codeword, early_here, fetch
  dq rax, pop_reg64
  dq rbx, pop_reg64
  dq rbx, mul_reg64
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "divmod". Jonesforth calls it "/mod" but % is widely recognized
  ; and has no special Forth significance.
  dq litstring, "/%", early_create, early_self_codeword, early_here, fetch
  dq rdx, rdx, xor_reg64_reg64
  dq rbx, pop_reg64
  dq rax, pop_reg64
  dq rbx, divmod_reg64
  dq rdx, push_reg64
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "eq". Jonesforth calls it "="; most languages would call it "==".
  dq litstring, "=", early_create, early_self_codeword, early_here, fetch
  dq rax, pop_reg64
  dq rbx, pop_reg64
  dq rbx, rax, cmp_reg64_reg64
  dq cc_equal, al, set_reg8_cc
  dq lit, 0x01, rax, and_reg64_imm8
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "ne". Jonesforth calls it "<>", but even most modern SQL dialects
  ; recognize C's legacy and allow "!=" these days. As someone who learned C
  ; in childhood, this is not actually a hard call for us.
  dq litstring, "!=", early_create, early_self_codeword, early_here, fetch
  dq rax, pop_reg64
  dq rbx, pop_reg64
  dq rbx, rax, cmp_reg64_reg64
  dq cc_not_equal, al, set_reg8_cc
  dq lit, 0x01, rax, and_reg64_imm8
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "gt".
  dq litstring, ">", early_create, early_self_codeword, early_here, fetch
  dq rax, pop_reg64
  dq rbx, pop_reg64
  dq rbx, rax, cmp_reg64_reg64
  dq cc_greater, al, set_reg8_cc
  dq lit, 0x01, rax, and_reg64_imm8
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "lt".
  dq litstring, "<", early_create, early_self_codeword, early_here, fetch
  dq rax, pop_reg64
  dq rbx, pop_reg64
  dq rbx, rax, cmp_reg64_reg64
  dq cc_less, al, set_reg8_cc
  dq lit, 0x01, rax, and_reg64_imm8
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "ge".
  dq litstring, ">=", early_create, early_self_codeword, early_here, fetch
  dq rax, pop_reg64
  dq rbx, pop_reg64
  dq rbx, rax, cmp_reg64_reg64
  dq cc_greater_equal, al, set_reg8_cc
  dq lit, 0x01, rax, and_reg64_imm8
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "le".
  dq litstring, "<=", early_create, early_self_codeword, early_here, fetch
  dq rax, pop_reg64
  dq rbx, pop_reg64
  dq rbx, rax, cmp_reg64_reg64
  dq cc_less_equal, al, set_reg8_cc
  dq lit, 0x01, rax, and_reg64_imm8
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "gt_unsigned".
  dq litstring, ">unsigned", early_create
  dq early_self_codeword, early_here, fetch
  dq rax, pop_reg64
  dq rbx, pop_reg64
  dq rbx, rax, cmp_reg64_reg64
  dq cc_above, al, set_reg8_cc
  dq lit, 0x01, rax, and_reg64_imm8
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "lt_unsigned".
  dq litstring, "<unsigned", early_create
  dq early_self_codeword, early_here, fetch
  dq rax, pop_reg64
  dq rbx, pop_reg64
  dq rbx, rax, cmp_reg64_reg64
  dq cc_below, al, set_reg8_cc
  dq lit, 0x01, rax, and_reg64_imm8
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "ge_unsigned".
  dq litstring, ">=unsigned", early_create
  dq early_self_codeword, early_here, fetch
  dq rax, pop_reg64
  dq rbx, pop_reg64
  dq rbx, rax, cmp_reg64_reg64
  dq cc_above_equal, al, set_reg8_cc
  dq lit, 0x01, rax, and_reg64_imm8
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "le_unsigned".
  dq litstring, "<=unsigned", early_create
  dq early_self_codeword, early_here, fetch
  dq rax, pop_reg64
  dq rbx, pop_reg64
  dq rbx, rax, cmp_reg64_reg64
  dq cc_below_equal, al, set_reg8_cc
  dq lit, 0x01, rax, and_reg64_imm8
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  dq litstring, "&", early_create, early_self_codeword, early_here, fetch
  dq rbx, pop_reg64
  dq rax, pop_reg64
  dq rbx, rax, and_reg64_reg64
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  dq litstring, "|", early_create, early_self_codeword, early_here, fetch
  dq rbx, pop_reg64
  dq rax, pop_reg64
  dq rbx, rax, or_reg64_reg64
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  dq litstring, "xor", early_create, early_self_codeword, early_here, fetch
  dq rbx, pop_reg64
  dq rax, pop_reg64
  dq rbx, rax, xor_reg64_reg64
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  dq litstring, "invert", early_create, early_self_codeword, early_here, fetch
  dq rax, pop_reg64
  dq rax, not_reg64
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  dq litstring, "lit", early_create, early_self_codeword, early_here, fetch
  dq lods64
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  dq litstring, "litstring", early_create, early_self_codeword
  dq early_here, fetch
  dq rsi, push_reg64
  dq rsi, rdi, mov_reg64_reg64
  dq rax, rax, xor_reg64_reg64
  dq rcx, rcx, xor_reg64_reg64
  dq rcx, not_reg64
  dq repnz_scas8
  dq rdi, rsi, mov_reg64_reg64
  dq lit, 7, rsi, add_reg64_imm8
  dq lit, 0xF8, rsi, and_reg64_imm8
  dq pack_next, lit, 8, packalign, early_here_store

  ; This is "store", but since we're finally in Forth we're no longer limited
  ; by flatassembler's syntax, so we call it by its usual Forth name.
  dq litstring, "!", early_create, early_self_codeword, early_here, fetch
  dq rbx, pop_reg64
  dq rax, pop_reg64
  dq rax, rbx, mov_indirect_reg64_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This is "fetch".
  dq litstring, "@", early_create, early_self_codeword, early_here, fetch
  dq rax, pop_reg64
  dq rax, rax, mov_reg64_indirect_reg64
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This is "addstore".
  dq litstring, "+!", early_create, early_self_codeword, early_here, fetch
  dq rbx, pop_reg64
  dq rax, pop_reg64
  dq rax, rbx, add_indirect_reg64_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This is "substore".
  dq litstring, "-!", early_create, early_self_codeword, early_here, fetch
  dq rbx, pop_reg64
  dq rax, pop_reg64
  dq rax, rbx, sub_indirect_reg64_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "store8". Jonesforth calls it "c!" but we categorically reject
  ; the use of letters that are meant to indicate byte sizes. It's unfriendly
  ; to newcomers. There's some risk that this name will be confused for
  ; meaning "store a value of 8", but that would not be a useful task, so
  ; hopefully it'll be okay.
  dq litstring, "8!", early_create, early_self_codeword, early_here, fetch
  dq rbx, pop_reg64
  dq rax, pop_reg64
  dq al, rbx, mov_indirect_reg64_reg8
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "fetch8".
  dq litstring, "8@", early_create, early_self_codeword, early_here, fetch
  dq rbx, pop_reg64
  dq rax, rax, xor_reg64_reg64
  dq rbx, al, mov_reg8_indirect_reg64
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "store16".
  dq litstring, "16!", early_create, early_self_codeword, early_here, fetch
  dq rbx, pop_reg64
  dq rax, pop_reg64
  dq ax, rbx, mov_indirect_reg64_reg16
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "fetch16".
  dq litstring, "16@", early_create, early_self_codeword, early_here, fetch
  dq rbx, pop_reg64
  dq rax, rax, xor_reg64_reg64
  dq rbx, ax, mov_reg16_indirect_reg64
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "store32".
  dq litstring, "32!", early_create, early_self_codeword, early_here, fetch
  dq rbx, pop_reg64
  dq rax, pop_reg64
  dq eax, rbx, mov_indirect_reg64_reg32
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "fetch32".
  dq litstring, "32@", early_create, early_self_codeword, early_here, fetch
  dq rbx, pop_reg64
  dq rax, rax, xor_reg64_reg64
  dq rbx, eax, mov_reg32_indirect_reg64
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "store_control_stack". Jonesforth calls it RSP!, which looks as
  ; if it's meant to be an Intel register name but is actually short for
  ; return stack pointer. There is no register by that name, it's a
  ; Forth-provided abstraction. That's super confusing, plus as discussed
  ; above we call it the control stack not the return stack, so we call it...
  dq litstring, "control!", 0, early_create, early_self_codeword
  dq early_here, fetch
  dq rbp, pop_reg64
  dq pack_next, lit, 8, packalign, early_here_store
  ; This was "fetch_control_stack".
  dq litstring, "control@", 0, early_create, early_self_codeword
  dq early_here, fetch
  dq rbp, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store
  ; This was "store_value_stack". Jonesforth calls it DSP!, for data stack
  ; pointer. Again, there's no Intel register by that name, and we call it the
  ; value stack, so...
  dq litstring, "value!", early_create, early_self_codeword
  dq early_here, fetch
  dq rsp, pop_reg64
  dq pack_next, lit, 8, packalign, early_here_store
  ; This was "fetch_value_stack".
  dq litstring, "value@", early_create, early_self_codeword
  dq early_here, fetch
  dq rsp, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  dq litstring, "memcopy", early_create, early_self_codeword
  dq early_here, fetch
  dq rsi, rdx, mov_reg64_reg64
  dq rcx, pop_reg64
  dq rdi, pop_reg64
  dq rsi, pop_reg64
  dq rep_movs8
  dq rdx, rsi, mov_reg64_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  dq litstring, "memmove", early_create, early_self_codeword
  dq early_here, fetch
  dq rsi, rdx, mov_reg64_reg64
  dq rcx, pop_reg64
  dq rdi, pop_reg64
  dq rsi, pop_reg64
  dq rsi, rax, mov_reg64_reg64
  dq rdi, rax, cmp_reg64_reg64
  dq lit, 4, cc_below, jmp_cc_rel_imm8
  dq rep_movs8
  dq lit, 16, jmp_rel_imm8
  dq rcx, rsi, add_reg64_reg64
  dq rsi, dec_reg64
  dq rcx, rdi, add_reg64_reg64
  dq rdi, dec_reg64
  dq std
  dq rep_movs8
  dq cld
  dq rdx, rsi, mov_reg64_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  dq litstring, "stringlen", early_create, early_self_codeword
  dq early_here, fetch
  dq rdi, pop_reg64
  dq rdi, rbx, mov_reg64_reg64
  dq rax, rax, xor_reg64_reg64
  dq rcx, rcx, xor_reg64_reg64
  dq rcx, not_reg64
  dq repnz_scas8
  dq rbx, rdi, sub_reg64_reg64
  dq lit, 1, rdi, sub_reg64_imm8
  dq rdi, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "reverse_stringlen".
  dq litstring, "reverse-stringlen", early_create, early_self_codeword
  dq early_here, fetch
  dq rdi, pop_reg64
  dq rdi, rbx, mov_reg64_reg64
  dq rax, rax, xor_reg64_reg64
  dq rcx, rcx, xor_reg64_reg64
  dq rcx, not_reg64
  dq std
  dq repnz_scas8
  dq cld
  dq rdi, rbx, sub_reg64_reg64
  dq lit, 1, rbx, sub_reg64_imm8
  dq rbx, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "reverse_padding_len".
  dq litstring, "reverse-padding-len", early_create, early_self_codeword
  dq early_here, fetch
  dq rdi, pop_reg64
  dq rdi, rbx, mov_reg64_reg64
  dq rax, rax, xor_reg64_reg64
  dq lit, 9, rcx, mov_reg64_imm32
  dq std
  dq repz_scas8
  dq cld
  dq rdi, rbx, sub_reg64_reg64
  dq lit, 1, rbx, sub_reg64_imm8
  dq rbx, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  dq litstring, "stringcmp", early_create, early_self_codeword
  dq early_here, fetch
  dq rsi, rdx, mov_reg64_reg64
  dq rsi, pop_reg64
  dq rdi, pop_reg64
  dq rbx, rbx, xor_reg64_reg64
  dq rax, rax, xor_reg64_reg64
  dq rsi, rcx, mov_reg64_indirect_reg64
  dq cl, al, mov_reg8_reg8
  dq cmps8
  dq lit, 15, cc_equal, jmp_cc_rel_imm8
  dq cc_above, bl, set_reg8_cc
  dq lit, 0, rbx, sbb_reg64_imm8
  dq rbx, push_reg64
  dq rdx, rsi, mov_reg64_reg64
  dq pack_next
  dq rax, rax, test_reg64_reg64
  dq lit, -28, cc_not_equal, jmp_cc_rel_imm8
  dq lit, 0, push_imm32_extended64
  dq rdx, rsi, mov_reg64_reg64
  dq pack_next
  dq lit, 8, packalign, early_here_store

  dq litstring, "branch", early_create, early_self_codeword, early_here, fetch
  ; Save the address we're packing to; it will be the label we jump to from
  ; zbranch.
  dq dup, unroll3
  dq rsi, rsi, add_reg64_indirect_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ; This was "zbranch".
  dq litstring, "0branch", early_create, early_self_codeword
  dq early_here, fetch
  dq rax, pop_reg64
  dq rax, rax, test_reg64_reg64
  ; Retrieve the saved address for the start of "branch". Compute the
  ; relative offset from the current address to it. This will be a negative
  ; number. Subtract an additional two bytes, to allow for the length of the
  ; jmp instruction.
  dq dup, lit, 4, roll, swap, sub, lit, 2, sub
  ; It's slightly counterintuitive that the condition is called cc_equal;
  ; that's a result of the condition names favoring cmp over test. While cmp
  ; simulates subtraction, test simulates bitwise AND. What we're testing is
  ; that the result is zero.
  dq cc_equal, jmp_cc_rel_imm8
  dq lods64
  dq pack_next, lit, 8, packalign, early_here_store

  dq litstring, "execute", early_create, early_self_codeword
  dq early_here, fetch
  dq rax, pop_reg64
  dq rax, jmp_abs_indirect_reg64
  dq lit, 8, packalign, early_here_store

  ; This was "sys_exit".
  ;
  ; This name is exactly eight bytes long. Don't even ask (go read litstring's
  ; code if you really need to know).
  dq litstring, "sys-exit", 0, early_create, early_self_codeword
  dq early_here, fetch
  dq lit, 60, rax, mov_reg64_imm64
  dq rdi, pop_reg64
  dq syscall
  dq hlt
  ; This one, uniquely, doesn't need to be followed by the "next" macro. It
  ; still needs alignment padding though.
  dq lit, 8, packalign, early_here_store

  ; This was "sys_write".
  dq litstring, "sys-write", early_create, early_self_codeword
  dq early_here, fetch
  dq rcx, pop_reg64
  dq rdx, pop_reg64
  dq rsi, push_reg64
  dq lit, 1, rax, mov_reg64_imm64
  dq lit, 1, rdi, mov_reg64_imm64
  dq rcx, rsi, mov_reg64_reg64
  dq syscall
  dq rsi, pop_reg64
  dq pack_next, lit, 8, packalign, early_here_store

 ; This was "sys_read".
  dq litstring, "sys-read", 0, early_create, early_self_codeword
  dq early_here, fetch
  dq rcx, pop_reg64
  dq rdx, pop_reg64
  dq rsi, push_reg64
  dq lit, 0, rax, mov_reg64_imm64
  dq lit, 0, rdi, mov_reg64_imm64
  dq rcx, rsi, mov_reg64_reg64
  dq syscall
  dq rsi, pop_reg64
  dq rax, push_reg64
  dq pack_next, lit, 8, packalign, early_here_store

  ;   This one has the honor of being the first word written in Forth. To do
  ; that, it needs to be able to look up the words we've defined so far, to
  ; reference them. We have a valid dictionary data structure, and early_find
  ; is able to search it, so we can do that.
  ;
  ;   One thing to be cautious of is that if we inadvertently reference a word
  ; by the wrong name, we'll get the copy that's statically included in the
  ; ELF, rather than the copy on the heap. Watch carefully for that, with all
  ; the lookups from here on.
  dq litstring, "emitstring", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "stringlen", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "sys-write", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "pow", early_create, early_docol_codeword
  dq early_here, fetch, lit, 8, packalign, early_here_store
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "-", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "*", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "branch", early_find, entry_to_execution_token, early_comma
  dq lit, -22*8, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "logfloor", 0, early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "2dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ">unsigned", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 6*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "*", early_find, entry_to_execution_token, early_comma
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "<", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 8*8, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "branch", early_find, entry_to_execution_token, early_comma
  dq lit, -45*8, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "logceil", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "2dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ">=unsigned", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 6*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "*", early_find, entry_to_execution_token, early_comma
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "<", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 8*8, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "branch", early_find, entry_to_execution_token, early_comma
  dq lit, -45*8, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "dotbase_unsigned"
  dq litstring, ".base-unsigned", early_create, early_docol_codeword
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "2dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "logfloor", 0, early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "2dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ">", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "branch", early_find, entry_to_execution_token, early_comma
  dq lit, 2*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "2dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "-", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 6, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "pow", early_find, entry_to_execution_token, early_comma
  dq litstring, "/%", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "/%", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 10, early_comma
  dq litstring, ">", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, "0", early_comma
  dq litstring, "branch", early_find, entry_to_execution_token, early_comma
  dq lit, 6*8, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 10, early_comma
  dq litstring, "-", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, "a", early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "value@", early_find, entry_to_execution_token, early_comma
  dq litstring, "emitstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "-", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 3*8, early_comma
  dq litstring, "branch", early_find, entry_to_execution_token, early_comma
  dq lit, -51*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "dotbase".
  dq litstring, ".base", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, ">", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 7*8, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, -1, early_comma
  dq litstring, "*", early_find, entry_to_execution_token, early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, "-", early_comma
  dq litstring, "emitstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, ".base-unsigned", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "dot".
  dq litstring, ".", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 10, early_comma
  dq litstring, ".base", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "dothex".
  dq litstring, ".hex", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 16, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, ".base-unsigned", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "dothex8".
  dq litstring, ".hex8", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 16, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 2, early_comma
  dq litstring, ".base-unsigned", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "dothex16".
  dq litstring, ".hex16", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 16, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, ".base-unsigned", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "dothex32".
  dq litstring, ".hex32", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 16, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, ".base-unsigned", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "dothex64".
  dq litstring, ".hex64", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 16, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 16, early_comma
  dq litstring, ".base-unsigned", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "crash", early_create, early_self_codeword, early_here, fetch
  dq hlt
  dq lit, 8, packalign, early_here_store

  ; This was "align_size".
  dq litstring, "align-size", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "-", early_find, entry_to_execution_token, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "/%", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "*", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; We could in theory call this 64pack, by analogy to 32! and so on, but it's
  ; kept like this to be more similar to reg64, imm64 etc.
  ;
  ; Also, unlike with ! we specify the size even for the 64-bit version,
  ; because the user of pack64 and its variants is always doing something
  ; where exact byte sizes are crucial.
  dq litstring, "pack64", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "pack32", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "32!", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "pack16", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "16!", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 2, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "pack8", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "8!", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "packstring", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "stringlen", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "memcopy", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "pack_raw_string".
  dq litstring, "pack-raw-string", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "stringlen", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "memcopy", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "packalign", early_create, early_docol_codeword
  dq litstring, "2dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "/%", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 8*8, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "branch", early_find, entry_to_execution_token, early_comma
  dq lit, -11*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "litpack64", early_create, early_self_codeword
  dq early_here, fetch
  dq lods64
  dq rax, push_reg64
  ; ("heap", modified "here")
  dq swap, litstring, "pack64", early_find, entry_to_execution_token
  ; (modified "here", "heap", execution token)
  dq swap, unroll3, pack_beforenext
  dq lit, 8, packalign, early_here_store

  dq litstring, "litpack32", early_create, early_self_codeword
  dq early_here, fetch
  dq lods64
  dq rax, push_reg64
  ; ("heap", modified "here")
  dq swap, litstring, "pack32", early_find, entry_to_execution_token
  ; (modified "here", "heap", execution token)
  dq swap, unroll3, pack_beforenext
  dq lit, 8, packalign, early_here_store

  dq litstring, "litpack16", early_create, early_self_codeword
  dq early_here, fetch
  dq lods64
  dq rax, push_reg64
  ; ("heap", modified "here")
  dq swap, litstring, "pack16", early_find, entry_to_execution_token
  ; (modified "here", "heap", execution token)
  dq swap, unroll3, pack_beforenext
  dq lit, 8, packalign, early_here_store

  ; This name is exactly eight bytes long. Don't even ask (go read litstring's
  ; code if you really need to know).
  dq litstring, "litpack8", 0, early_create, early_self_codeword
  dq early_here, fetch
  dq lods64
  dq rax, push_reg64
  ; ("heap", modified "here")
  dq swap, litstring, "pack8", early_find, entry_to_execution_token
  ; (modified "here", "heap", execution token)
  dq swap, unroll3, pack_beforenext
  dq lit, 8, packalign, early_here_store

  dq litstring, "unpack64", 0, early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "unpack32", 0, early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "32@", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "unpack16", 0, early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "16@", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 2, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "unpack8", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "8@", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "unpackalign", early_create, early_docol_codeword
  dq litstring, "align-size", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; The flatassembler names of all these were the same, but without the
  ; colons, and with underscores instead of hyphens.
  dq litstring, ":rax", early_keyword
  dq litstring, ":rcx", early_keyword
  dq litstring, ":rdx", early_keyword
  dq litstring, ":rbx", early_keyword
  dq litstring, ":rsp", early_keyword
  dq litstring, ":rbp", early_keyword
  dq litstring, ":rsi", early_keyword
  dq litstring, ":rdi", early_keyword
  dq litstring, ":r8", early_keyword
  dq litstring, ":r9", early_keyword
  dq litstring, ":r10", early_keyword
  dq litstring, ":r11", early_keyword
  dq litstring, ":r12", early_keyword
  dq litstring, ":r13", early_keyword
  dq litstring, ":r14", early_keyword
  dq litstring, ":r15", early_keyword
  dq litstring, ":eax", early_keyword
  dq litstring, ":ecx", early_keyword
  dq litstring, ":edx", early_keyword
  dq litstring, ":ebx", early_keyword
  dq litstring, ":esp", early_keyword
  dq litstring, ":ebp", early_keyword
  dq litstring, ":esi", early_keyword
  dq litstring, ":edi", early_keyword
  dq litstring, ":ax", early_keyword
  dq litstring, ":cx", early_keyword
  dq litstring, ":dx", early_keyword
  dq litstring, ":bx", early_keyword
  dq litstring, ":sp", early_keyword
  dq litstring, ":bp", early_keyword
  dq litstring, ":si", early_keyword
  dq litstring, ":di", early_keyword
  dq litstring, ":al", early_keyword
  dq litstring, ":cl", early_keyword
  dq litstring, ":dl", early_keyword
  dq litstring, ":bl", early_keyword
  dq litstring, ":ah", early_keyword
  dq litstring, ":ch", early_keyword
  dq litstring, ":dh", early_keyword
  dq litstring, ":bh", early_keyword
  dq litstring, ":cc-overflow", early_keyword
  dq litstring, ":cc-no-overflow", early_keyword
  dq litstring, ":cc-below", early_keyword
  dq litstring, ":cc-above-equal", early_keyword
  dq litstring, ":cc-equal", early_keyword
  dq litstring, ":cc-not-equal", early_keyword
  dq litstring, ":cc-below-equal", early_keyword
  dq litstring, ":cc-above", early_keyword
  dq litstring, ":cc-sign", 0, early_keyword
  dq litstring, ":cc-not-sign", early_keyword
  dq litstring, ":cc-even", 0, early_keyword
  dq litstring, ":cc-odd", early_keyword
  dq litstring, ":cc-less", 0, early_keyword
  dq litstring, ":cc-greater-equal", early_keyword
  dq litstring, ":cc-less-equal", early_keyword
  dq litstring, ":cc-greater", early_keyword

  dq litstring, "reg64", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":rax", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":rcx", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":rdx", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 2, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":rbx", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 3, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":rsp", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":rbp", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":rsi", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 6, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":rdi", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 7, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, "Paramete", early_comma, lit, "r to reg", early_comma
  dq lit, "64 is no", early_comma, lit, "t a reg6", early_comma
  dq lit, "4.", early_comma
  dq litstring, "emitstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "sys-exit", 0, early_find, entry_to_execution_token
  dq early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "extrareg64", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":r8", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":r9", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":r10", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 2, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":r11", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 3, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":r12", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":r13", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":r14", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 6, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":r15", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 7, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, "Paramete", early_comma, lit, "r to ext", early_comma
  dq lit, "rareg64 ", early_comma, lit, "is not a", early_comma
  dq lit, "n extrar", early_comma, lit, "eg64.", early_comma
  dq litstring, "emitstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "sys-exit", 0, early_find, entry_to_execution_token
  dq early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "reg32", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":eax", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":ecx", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":edx", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 2, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":ebx", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 3, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":esp", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":ebp", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":esi", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 6, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":edi", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 7, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, "Paramete", early_comma, lit, "r to reg", early_comma
  dq lit, "32 is no", early_comma, lit, "t a reg3", early_comma
  dq lit, "2.", early_comma
  dq litstring, "emitstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "sys-exit", 0, early_find, entry_to_execution_token
  dq early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "reg16", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":ax", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":cx", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":dx", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 2, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":bx", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 3, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":sp", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":bp", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":si", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 6, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":di", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 7, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, "Paramete", early_comma, lit, "r to reg", early_comma
  dq lit, "16 is no", early_comma, lit, "t a reg1", early_comma
  dq lit, "6.", early_comma
  dq litstring, "emitstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "sys-exit", 0, early_find, entry_to_execution_token
  dq early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "reg8", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":al", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":cl", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":dl", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 2, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":bl", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 3, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":ah", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":ch", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":dh", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 6, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":bh", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 7, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, "Paramete", early_comma, lit, "r to reg", early_comma
  dq lit, "8 is not", early_comma, lit, " a reg8.", early_comma
  dq lit, 0, early_comma
  dq litstring, "emitstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "sys-exit", 0, early_find, entry_to_execution_token
  dq early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "scalefield", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 2, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 2, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 3, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, "Paramete", early_comma, lit, "r to sca", early_comma
  dq lit, "lefield ", early_comma, lit, "is not 1", early_comma
  dq lit, ", 2, 4, ", early_comma, lit, "or 8.", early_comma
  dq litstring, "emitstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "sys-exit", 0, early_find, entry_to_execution_token
  dq early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "conditioncode", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":cc-overflow", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":cc-no-overflow", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":cc-below", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 2, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":cc-above-equal", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 3, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":cc-equal", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":cc-not-equal", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":cc-below-equal", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 6, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":cc-above", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 7, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":cc-sign", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":cc-not-sign", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 9, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":cc-even", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 10, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":cc-odd", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 11, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":cc-less", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 12, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":cc-greater-equal", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 13, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":cc-less-equal", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 14, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":cc-greater", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 15, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, "Paramete", early_comma, lit, "r to con", early_comma
  dq lit, "ditionco", early_comma, lit, "de is no", early_comma
  dq lit, "t a cond", early_comma, lit, "ition co", early_comma
  dq lit, "de.", early_comma
  dq litstring, "emitstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "sys-exit", 0, early_find, entry_to_execution_token
  dq early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "rex_w".
  dq litstring, "rex-w", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x48, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "rex_b".
  dq litstring, "rex-b", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x41, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "rex_wb".
  dq litstring, "rex-wb", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x49, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "opcodereg", early_create, early_docol_codeword
  dq litstring, "|", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "opcodecc", 0, early_create, early_docol_codeword
  dq litstring, "|", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "modrm", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "*", early_find, entry_to_execution_token, early_comma
  dq litstring, "|", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 64, early_comma
  dq litstring, "*", early_find, entry_to_execution_token, early_comma
  dq litstring, "|", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "sib", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "*", early_find, entry_to_execution_token, early_comma
  dq litstring, "|", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 64, early_comma
  dq litstring, "*", early_find, entry_to_execution_token, early_comma
  dq litstring, "|", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "addressing_reg64".
  dq litstring, "addressing-reg64", 0, early_create, early_docol_codeword
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 3, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "modrm", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "addressing_indirect_reg64".
  dq litstring, "addressing-indirect-reg64", early_create
  dq early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":rbp", early_find, entry_to_execution_token, early_comma
  dq litstring, "!=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 23*8, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":rsp", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "modrm", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 8*8, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, ":rsp", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "sib", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, "R/M para", early_comma, lit, "meter to", early_comma
  dq lit, " address", early_comma, lit, "ing-indi", early_comma
  dq lit, "rect-reg", early_comma, lit, "64 is :r", early_comma
  dq lit, "bp.", early_comma
  dq litstring, "emitstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "sys-exit", 0, early_find, entry_to_execution_token
  dq early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "addressing_disp8_reg64".
  dq litstring, "addressing-disp8-reg64", early_create
  dq early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":rsp", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "modrm", early_find, entry_to_execution_token, early_comma
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 8*8, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, ":rsp", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "sib", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "addressing_disp32_reg64".
  dq litstring, "addressing-disp32-reg64", early_create
  dq early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":rsp", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 2, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "modrm", early_find, entry_to_execution_token, early_comma
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 8*8, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, ":rsp", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "sib", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack32", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "addressing_indexed_reg64".
  dq litstring, "addressing-indexed-reg64", 0, early_create
  dq early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ":rbp", early_find, entry_to_execution_token, early_comma
  dq litstring, "!=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 23*8, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 6, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "modrm", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "scalefield", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "sib", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, "Base par", early_comma, lit, "ameter t", early_comma
  dq lit, "o addres", early_comma, lit, "sing-ind", early_comma
  dq lit, "exed-reg", early_comma, lit, "64 is :r", early_comma
  dq lit, "bp.", early_comma
  dq litstring, "emitstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "sys-exit", 0, early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "addressing_disp8_indexed_reg64".
  dq litstring, "addressing-disp8-indexed-reg64", early_create
  dq early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 6, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 7, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "modrm", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "scalefield", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "sib", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "addressing_reg8".
  dq litstring, "addressing-reg8", early_create, early_docol_codeword
  dq litstring, "reg8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 3, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "modrm", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "cld", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xFC, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "std", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xFD, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_reg64_imm32".
  dq litstring, "mov-reg64-imm32", early_create, early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xC7, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-reg64", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack32", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_reg64_imm64".
  dq litstring, "mov-reg64-imm64", early_create, early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xB8, early_comma
  dq litstring, "opcodereg", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack64", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_extrareg64_imm64".
  dq litstring, "mov-extrareg64-imm64", early_create, early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-wb", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "extrareg64", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xB8, early_comma
  dq litstring, "opcodereg", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack64", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_reg64_reg64".
  dq litstring, "mov-reg64-reg64", early_create, early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x89, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-reg64", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_indirect_reg64_reg64".
  dq litstring, "mov-indirect-reg64-reg64", 0, early_create
  dq early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x89, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-indirect-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_disp8_reg64_reg64".
  dq litstring, "mov-disp8-reg64-reg64", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x89, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-disp8-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_reg64_indirect_reg64".
  dq litstring, "mov-reg64-indirect-reg64", 0, early_create
  dq early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x8B, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-indirect-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_reg64_disp8_reg64".
  dq litstring, "mov-reg64-disp8-reg64", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x8B, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-disp8-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_reg64_disp32_reg64".
  dq litstring, "mov-reg64-disp32-reg64", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x89, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-disp32-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_reg64_indexed_reg64".
  dq litstring, "mov-reg64-indexed-reg64", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x8B, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-indexed-reg64", 0, early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_indexed_reg64_reg64".
  dq litstring, "mov-indexed-reg64-reg64", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x89, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-indexed-reg64", 0, early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_indirect_reg64_reg32".
  dq litstring, "mov-indirect-reg64-reg32", 0, early_create
  dq early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x89, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg32", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-indirect-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_indirect_reg64_reg32".
  dq litstring, "mov-disp8-reg64-reg32", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x89, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg32", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-disp8-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ;; This was "mov_reg32_indirect_reg64".
  dq litstring, "mov-reg32-indirect-reg64", 0, early_create
  dq early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x8B, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg32", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-indirect-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_reg32_disp8_reg64".
  dq litstring, "mov-reg32-disp8-reg64", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x8B, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg32", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-disp8-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_indirect_reg64_reg16".
  dq litstring, "mov-indirect-reg64-reg16", 0, early_create
  dq early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x66, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x89, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg16", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-indirect-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_disp8_reg64_reg16".
  dq litstring, "mov-disp8-reg64-reg16", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x66, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x89, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg16", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-disp8-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_reg16_indirect_reg64".
  dq litstring, "mov-reg16-indirect-reg64", 0, early_create
  dq early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x66, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x8B, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg16", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-indirect-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_reg16_disp8_reg64".
  dq litstring, "mov-reg16-disp8-reg64", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x66, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x8B, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg16", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-disp8-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_indirect_reg64_reg8".
  dq litstring, "mov-indirect-reg64-reg8", early_create, early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x88, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg8", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-indirect-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_disp8_reg64_reg8".
  dq litstring, "mov-disp8-reg64-reg8", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x88, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-disp8-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_reg8_indirect_reg64".
  dq litstring, "mov-reg8-indirect-reg64", early_create, early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x8A, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg8", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-indirect-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_reg8_disp8_reg64".
  dq litstring, "mov-reg8-disp8-reg64", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x8A, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-disp8-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mov_reg8_reg8".
  dq litstring, "mov-reg8-reg8", early_create, early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x88, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg8", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-reg8", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "lea_reg64_disp8_reg64".
  dq litstring, "lea-reg64-disp8-reg64", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x8D, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-disp8-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "lea_reg64_disp32_reg64".
  dq litstring, "lea-reg64-disp32-reg64", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x8D, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-disp32-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "lea_reg64_indexed_reg64".
  dq litstring, "lea-reg64-indexed-reg64", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x8D, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-indexed-reg64", 0, early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "lea_reg64_disp8_indexed_reg64".
  dq litstring, "lea-reg64-disp8-indexed-reg64", early_create
  dq early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 6, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x8D, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 6, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 3, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 3, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-disp8-indexed-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "push_reg64".
  dq litstring, "push-reg64", early_create, early_docol_codeword
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x50, early_comma
  dq litstring, "opcodereg", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "push_extrareg64".
  dq litstring, "push-extrareg64", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-b", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "extrareg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x50, early_comma
  dq litstring, "opcodereg", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "pop_reg64".
  dq litstring, "pop-reg64", early_create, early_docol_codeword
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x58, early_comma
  dq litstring, "opcodereg", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "pop_extrareg64".
  dq litstring, "pop-extrareg64", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-b", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "extrareg64", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x58, early_comma
  dq litstring, "opcodereg", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was push_imm32_extended64".
  dq litstring, "push-imm32-extended64", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x68, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack32", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "movs8", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xA4, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "movs16", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x66, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xA5, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "movs32", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xA5, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "movs64", early_create, early_docol_codeword
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xA5, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "rep_movs8".
  dq litstring, "rep-movs8", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF3, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xA4, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "rep_movs16".
  dq litstring, "rep-movs16", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF3, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x66, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xA5, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "rep_movs32".
  dq litstring, "rep-movs32", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF3, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xA5, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "rep_movs64".
  dq litstring, "rep-movs64", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF3, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xA5, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  dq litstring, "lods8", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAC, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "lods16", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x66, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAD, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "lods32", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAD, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "lods64", early_create, early_docol_codeword
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAD, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "rep_lods8".
  dq litstring, "rep-lods8", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF3, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAC, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "rep_lods16".
  dq litstring, "rep-lods16", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF3, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x66, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAD, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "rep_lods32".
  dq litstring, "rep-lods32", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF3, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAD, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "rep_lods64".
  dq litstring, "rep-lods64", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF3, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAD, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "stos8", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAA, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "stos16", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x66, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAB, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "stos32", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAB, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "stos64", early_create, early_docol_codeword
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAB, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "rep_stos8".
  dq litstring, "rep-stos8", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF3, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAA, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "rep_stos16".
  dq litstring, "rep-stos16", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF3, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x66, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAB, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "rep_stos32".
  dq litstring, "rep-stos32", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF3, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAB, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "rep_stos64".
  dq litstring, "rep-stos64", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF3, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAB, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "cmps8", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xA6, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "cmps16", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x66, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xA7, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "cmps32", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xA7, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "cmps64", early_create, early_docol_codeword
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xA7, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "repz_cmps8".
  dq litstring, "repz-cmps8", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF3, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xA6, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "repz_cmps16".
  dq litstring, "repz-cmps16", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF3, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x66, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xA7, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "repz_cmps32".
  dq litstring, "repz-cmps32", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF3, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xA7, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "repz_cmps64".
  dq litstring, "repz-cmps64", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF3, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xA7, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "repnz_cmps8".
  dq litstring, "repnz-cmps8", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF2, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xA6, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "repnz_cmps16".
  dq litstring, "repnz-cmps16", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF2, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x66, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xA7, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "repnz_cmps32".
  dq litstring, "repnz-cmps32", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF2, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xA7, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "repnz_cmps64".
  dq litstring, "repnz-cmps64", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF2, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xA7, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "scas8", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAE, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "scas16", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x66, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAF, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "scas32", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAF, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "scas64", early_create, early_docol_codeword
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAF, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "repz_scas8".
  dq litstring, "repz-scas8", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF3, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAE, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "repz_scas16".
  dq litstring, "repz-scas16", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF3, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x66, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAF, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "repz_scas32".
  dq litstring, "repz-scas32", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF3, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAF, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "repz_scas64".
  dq litstring, "repz-scas64", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF3, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAF, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "repnz_scas8".
  dq litstring, "repnz-scas8", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF2, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAE, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "repnz_scas16".
  dq litstring, "repnz-scas16", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF2, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x66, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAF, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "repnz_scas32".
  dq litstring, "repnz-scas32", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF2, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAF, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "repnz_scas64".
  dq litstring, "repnz-scas64", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF2, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xAF, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "add_reg64_reg64".
  dq litstring, "add-reg64-reg64", early_create, early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x01, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-reg64", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "add_indirect_reg64_reg64".
  dq litstring, "add-indirect-reg64-reg64", 0, early_create
  dq early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x01, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-indirect-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "add_reg64_indirect_reg64".
  dq litstring, "add-reg64-indirect-reg64", 0, early_create
  dq early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x03, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-indirect-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "add_reg64_imm8".
  dq litstring, "add-reg64-imm8", early_create, early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x83, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-reg64", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "sub_reg64_reg64".
  dq litstring, "sub-reg64-reg64", early_create, early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x2B, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-reg64", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "sub_indirect_reg64_reg64".
  dq litstring, "sub-indirect-reg64-reg64", 0, early_create
  dq early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x2B, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-indirect-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "sub_reg64_imm8".
  dq litstring, "sub-reg64-imm8", early_create, early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x83, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-reg64", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "sbb_reg64_imm8".
  dq litstring, "sbb-reg64-imm8", early_create, early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x83, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 3, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-reg64", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "mul_reg64".
  dq litstring, "mul-reg64", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF7, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-reg64", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "divmod_reg64".
  dq litstring, "divmod-reg64", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF7, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 6, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-reg64", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "idivmod_reg64".
  dq litstring, "idivmod-reg64", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF7, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 7, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-reg64", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "inc_reg64".
  dq litstring, "inc-reg64", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xFF, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-reg64", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "dec_reg64".
  dq litstring, "dec-reg64", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xFF, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-reg64", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "and_reg64_reg64".
  dq litstring, "and-reg64-reg64", early_create, early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x23, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-reg64", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "and_reg64_imm8".
  dq litstring, "and-reg64-imm8", early_create, early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x83, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-reg64", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "or_reg64_reg64".
  dq litstring, "or-reg64-reg64", early_create, early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x0B, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-reg64", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "or_reg64_imm8".
  dq litstring, "or-reg64-imm8", early_create, early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x83, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-reg64", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "xor_reg64_reg64".
  dq litstring, "xor-reg64-reg64", early_create, early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x33, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-reg64", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "not_reg64".
  dq litstring, "not-reg64", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF7, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 2, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-reg64", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "cmp_reg64_reg64".
  dq litstring, "cmp-reg64-reg64", early_create, early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x3B, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-reg64", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "test_reg64_reg64".
  dq litstring, "test-reg64-reg64", 0, early_create, early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "rex-w", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x85, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg64", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-reg64", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "set_reg8_cc".
  dq litstring, "set-reg8-cc", early_create, early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x0F, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "conditioncode", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x90, early_comma
  dq litstring, "opcodecc", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "reg8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 3, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "modrm", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "jmp_cc_rel_imm8".
  dq litstring, "jmp-cc-rel-imm8", early_create, early_docol_codeword
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "conditioncode", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x70, early_comma
  dq litstring, "opcodecc", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "jmp_cc_rel_imm32".
  dq litstring, "jmp-cc-rel-imm32", 0, early_create, early_docol_codeword
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x0F, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "conditioncode", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x70, early_comma
  dq litstring, "opcodecc", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack32", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "jmp_abs_indirect_reg64".
  dq litstring, "jmp-abs-indirect-reg64", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xFF, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "addressing-indirect-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "jmp_rel_imm8".
  dq litstring, "jmp-rel-imm8", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xEB, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "jmp_rel_imm32".
  dq litstring, "jmp-rel-imm32", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xE9, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack32", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "syscall", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x0F, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x05, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "hlt", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xF4, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "pack_next".
  dq litstring, "pack-next", early_create, early_docol_codeword
  dq litstring, "lods64", early_find, entry_to_execution_token, early_comma
  dq litstring, ":rax", early_find, entry_to_execution_token, early_comma
  dq litstring, "jmp-abs-indirect-reg64", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "pack_beforenext".
  dq litstring, "pack-beforenext", early_create, early_docol_codeword
  dq litstring, ":rax", early_find, entry_to_execution_token, early_comma
  dq litstring, "mov-reg64-imm64", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, ":rax", early_find, entry_to_execution_token, early_comma
  dq litstring, "jmp-abs-indirect-reg64", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "pack_pushcontrol".
  dq litstring, "pack-pushcontrol", 0, early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, ":rbp", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, -8, early_comma
  dq litstring, ":rbp", early_find, entry_to_execution_token, early_comma
  dq litstring, "lea-reg64-disp8-reg64", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, ":rbp", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "mov-disp8-reg64-reg64", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "pack_popcontrol".
  dq litstring, "pack-popcontrol", early_create, early_docol_codeword
  dq litstring, ":rbp", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "mov-reg64-disp8-reg64", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, ":rbp", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, ":rbp", early_find, entry_to_execution_token, early_comma
  dq litstring, "lea-reg64-disp8-reg64", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "entry_to_execution_token".
  dq litstring, "entry-to-execution-token", 0, early_create
  dq early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 10, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "stringlen", early_find, entry_to_execution_token, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 7, early_comma
  dq litstring, "invert", early_find, entry_to_execution_token, early_comma
  dq litstring, "&", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "execution_token_to_entry".
  dq litstring, "execution-token-to-entry", 0, early_create
  dq early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "-", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "reverse-padding-len", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "-", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "reverse-stringlen", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "-", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 9, early_comma
  dq litstring, "-", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "fetch_entry_flags".
  dq litstring, "entry-flags@", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xFF, early_comma
  dq litstring, "&", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "store_entry_flags".
  dq litstring, "entry-flags!", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xFF, early_comma
  dq litstring, "&", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0xFFFFFFFFFFFFFF00, early_comma
  dq litstring, "&", early_find, entry_to_execution_token, early_comma
  dq litstring, "|", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "entry_to_name".
  dq litstring, "entry-to-name", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 10, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "find_in".
  dq litstring, "find-in", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 4*8, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "entry-flags@", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x80, early_comma
  dq litstring, "&", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x80, early_comma
  dq litstring, "!=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 8*8, early_comma
  dq litstring, "2dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 10, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "stringcmp", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 4*8, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "branch", early_find, entry_to_execution_token, early_comma
  dq lit, -28*8, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "next_newer_entry_in".
  dq litstring, "next-newer-entry-in", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 4*8, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "2dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "!=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 4*8, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "branch", early_find, entry_to_execution_token, early_comma
  dq lit, -16*8, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "guess_entry_end_in".
  dq litstring, "guess-entry-end-in", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "entry-flags@", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x01, early_comma
  dq litstring, "&", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x01, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 6*8, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "next-newer-entry-in", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 4*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "show_hex_between".
  dq litstring, "show-hex-between", 0, early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "entry-to-execution-token", 0, early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "2dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ">=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 4*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, " ", early_comma
  dq litstring, "emitstring", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, ".hex64", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "branch", early_find, entry_to_execution_token, early_comma
  dq lit, -17*8, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "show_source_between".
  dq litstring, "show-source-between", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "entry-to-execution-token", 0, early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "2dup", early_find, entry_to_execution_token, early_comma
  dq litstring, ">=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 4*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "execution-token-to-entry", 0, early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "entry-to-name", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, " ", early_comma
  dq litstring, "emitstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "emitstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, "lit", early_comma
  dq litstring, "stringcmp", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 57*8, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, "litpack8", early_comma, lit, 0, early_comma
  dq litstring, "stringcmp", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 50*8, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, "litpack1", early_comma, lit, "6", early_comma
  dq litstring, "stringcmp", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 43*8, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, "litpack3", early_comma, lit, "2", early_comma
  dq litstring, "stringcmp", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 36*8, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, "litpack6", early_comma, lit, "4", early_comma
  dq litstring, "stringcmp", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 29*8, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, "branch", early_comma
  dq litstring, "stringcmp", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 23*8, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, "zbranch", early_comma
  dq litstring, "stringcmp", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 17*8, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, "0branch", early_comma
  dq litstring, "stringcmp", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 11*8, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, "litstrin", early_comma, lit, "g", early_comma
  dq litstring, "stringcmp", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 16*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "branch", early_find, entry_to_execution_token, early_comma
  dq lit, -82*8, early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, " ", early_comma
  dq litstring, "emitstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, ".", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "branch", early_find, entry_to_execution_token, early_comma
  dq lit, -94*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, " ", early_comma
  dq litstring, "emitstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x22, early_comma
  dq litstring, "value@", early_find, entry_to_execution_token, early_comma
  dq litstring, "emitstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "emitstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x22, early_comma
  dq litstring, "value@", early_find, entry_to_execution_token, early_comma
  dq litstring, "emitstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "stringlen", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "/%", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "*", early_find, entry_to_execution_token, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "branch", early_find, entry_to_execution_token, early_comma
  dq lit, -126*8, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; This was "show_source_or_hex_between".
  dq litstring, "show-source-or-hex-between", early_create
  dq early_docol_codeword
  dq litstring, "2dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 2*8, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "entry-to-execution-token", 0, early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "-", early_find, entry_to_execution_token, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 3*8, early_comma
  dq litstring, "show-hex-between", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, "show-source-between", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ;;; Now here we have some "original" words that rely on the heap and don't
  ;;; have flatassembler-based equivalents.

  ;   This has the distinction of having been written in Evocation, in
  ; interpret.e, and then back-ported to flatassembler. This is the version
  ; that makes clever use of memmove, if we do say so ourselves, to handle
  ; immediate-mode string literals as input.
  ;
  ; In:
  ;   name string
  dq litstring, "create", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "stringlen", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "here", early_find, entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 10, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "memmove", early_find, entry_to_execution_token, early_comma
  dq litstring, "here", early_find, entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "latest", early_find, entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack64", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "pack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "packalign", early_find, entry_to_execution_token, early_comma
  dq litstring, "here", early_find, entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "latest", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "here", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; In:
  ;   value to append to current word-in-progress
  dq litstring, ",", early_create, early_docol_codeword
  dq litstring, "here", early_find, entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack64", early_find, entry_to_execution_token, early_comma
  dq litstring, "here", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "self-codeword", early_create, early_docol_codeword
  dq litstring, "here", early_find, entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, ",", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; In:
  ;   address for new variable word to return
  ;   name string
  dq litstring, "variable", 0, early_create, early_docol_codeword
  dq litstring, "create", early_find, entry_to_execution_token, early_comma
  dq litstring, "self-codeword", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "here", early_find, entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, ":rax", early_find, entry_to_execution_token, early_comma
  dq litstring, "mov-reg64-imm64", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, ":rax", early_find, entry_to_execution_token, early_comma
  dq litstring, "push-reg64", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "pack-next", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "packalign", early_find, entry_to_execution_token, early_comma
  dq litstring, "here", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ;   Allocates bytes on the heap by incrementing the global "here" pointer.
  ; The "here" pointer is kept aligned to an 8-byte boundary, regardless of
  ; the size requested.
  ;
  ;   This does not create dictionary entries, it's just a raw memory
  ; interface. It's suitable for allocating data or scratch space.
  ;
  ; In:
  ;   size in bytes
  ; Out:
  ;   pointer to start
  dq litstring, "allocate", 0, early_create, early_docol_codeword
  dq litstring, "here", early_find, entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  ; (size, here value, here value)
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "packalign", early_find, entry_to_execution_token, early_comma
  dq litstring, "here", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ;   Takes a string pointer which must be equal to the current value of
  ; "here", ; indicating it is in the temporary allocation space, and
  ; allocates it permanently by advancing "here" to the next aligned address
  ; after it.
  ;
  ; In:
  ;   string pointer
  ; Out:
  ;   string pointer
  dq litstring, "allocate-string", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "stringlen", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "packalign", early_find, entry_to_execution_token, early_comma
  dq litstring, "here", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; In:
  ;   pointer to buffer metadata
  ; Out:
  ;   pointer to buffer "physical-start" field
  dq litstring, "buffer-physical-start", early_create, early_docol_codeword
  ; The physical-start field happens to be the first thing in the metadata, so
  ; this is an nop, but it still exists as a word because having it reduces
  ; confusion.
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; In:
  ;   pointer to buffer metadata
  ; Out:
  ;   pointer to buffer "physical-length" field
  dq litstring, "buffer-physical-length", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; In:
  ;   pointer to buffer metadata
  ; Out:
  ;   pointer to buffer "logical-start" field
  dq litstring, "buffer-logical-start", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 2*8, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; In:
  ;   pointer to buffer metadata
  ; Out:
  ;   pointer to buffer "logical-length" field
  dq litstring, "buffer-logical-length", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 3*8, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; In:
  ;   pointer to input buffer metadata
  ; Out:
  ;   pointer to input buffer "refill" field
  dq litstring, "input-buffer-refill", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4*8, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; In:
  ;   pointer to input buffer metadata
  ; Out:
  ;   pointer to input buffer "next-source" field
  dq litstring, "input-buffer-next-source", 0, early_create
  dq early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ;   Given an initialized buffer (input or otherwise), sets its logical-start
  ; and logical-length fields to indicate the buffer is empty. This relies on
  ; the buffer having a backing store attached, but does not alter the backing
  ; store or its contents.
  ;
  ; In:
  ;   pointer to buffer metadata
  dq litstring, "clear-buffer", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "buffer-physical-start", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  ; (address of backing store, metadata pointer)
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  ; (metadata pointer, address of backing store, metadata pointer)
  dq litstring, "buffer-logical-start", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "buffer-logical-length", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ;   Sets all fields in an input buffer metadata structure to zero,
  ; effectively detaching and leaking any backing store that had been attached
  ; to it. Suitable for use during initialization.
  ;
  ; In:
  ;   pointer to input buffer metadata
  dq litstring, "zero-input-buffer-metadata", early_create
  dq early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "buffer-physical-start", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "buffer-physical-length", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "buffer-logical-start", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "buffer-logical-length", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "input-buffer-refill", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  ; Notice the absence of a dup this time.
  dq litstring, "input-buffer-next-source", 0, early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ;   Allocates buffer metadata, with no backing store attached. Initializes
  ; the metadata to all zeroes.
  ;
  ; Out:
  ;   pointer to buffer metadata
  dq litstring, "allocate-input-buffer-metadata", early_create
  dq early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 6*8, early_comma
  dq litstring, "allocate", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "zero-input-buffer-metadata", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ;   Allocates input buffer metadata and a backing store, in one operation.
  ; Points the metadata to the backing store.
  ;
  ; In:
  ;   buffer capacity in bytes
  ; Out:
  ;   pointer to buffer metadata
  dq litstring, "allocate-input-buffer", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 6*8, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "allocate", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "zero-input-buffer-metadata", early_find
  dq entry_to_execution_token, early_comma
  ; (capacity in bytes, metadata pointer)
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 6*8, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  ; (capacity in bytes, metadata pointer, metadata pointer, physical start)
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "buffer-physical-start", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  ; (capacity in bytes, metadata pointer)
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "3unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "buffer-physical-length", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  ; (metadata pointer)
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "clear-buffer", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ;   Sets the backing store of an input buffer to point at a null-teriminated
  ; string and read from it.
  ;
  ; In:
  ;   pointer to buffer metadata
  ;   pointer to string
  dq litstring, "attach-string-to-input-buffer", early_create
  dq early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  ; (string pointer, metadata pointer)
  dq litstring, "2dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "buffer-physical-start", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  ; (string pointer, metadata pointer)
  dq litstring, "2dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "buffer-logical-start", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  ; (string pointer, metadata pointer)
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "stringlen", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  ; (string length, metadata pointer)
  dq litstring, "2dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "buffer-physical-length", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  ; (string length, metadata pointer)
  dq litstring, "buffer-logical-length", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ;;; Here we have some imperative code that runs immediately, to initialize
  ;;; some runtime data structures.

  ; First, we insert a metadata word header to delimit the space. Otherwise
  ; "describe" would crash when attempting to describe
  ; "attach-string-to-input-buffer".
  ; (heap pointer)
  dq litstring, "main-input-buffer-metadata", early_create
  dq litstring, "main-input-buffer-metadata", early_find
  dq lit, 1, store_entry_flags

  ; Having done that, now we do the runtime allocation.
  ; (heap pointer)
  dq litstring, "allocate-input-buffer-metadata", early_find
  dq entry_to_execution_token, execute

  ; We also initialize the metadata here, pointing it to the boot source as
  ; its backing store. We could do that later, but it's convenient to do it
  ; here.
  ; (heap pointer, metadata pointer)
  dq swap, litstring, "attach-string-to-input-buffer", early_find
  ; (metadata pointer, heap pointer, entry pointer)
  dq entry_to_execution_token, swap, unroll3
  ; (heap pointer, metadata pointer, execution token)
  dq swap, dup, unroll3
  ; (heap pointer, metadata pointer, execution token, metadata pointer)
  dq lit, boot_source, roll3
  ; (heap pointer, metadata pointer, metadata pointer, string pointer,
  ;  execution token)
  dq execute

  ; Finally, we define the named variable "main-input-buffer" so we can find
  ; it again.
  ; (heap pointer, metadata pointer)
  dq swap, litstring, "variable", 0, early_find, entry_to_execution_token
  ; (metadata pointer, heap pointer, execution token)
  dq swap, unroll3
  ; (heap pointer, metadata pointer, execution token)
  dq litstring, "main-input-buffer", swap
  ; (heap pointer, metadata pointer, string pointer, execution token)
  dq execute


  ; (metadata pointer)
  dq litstring, "consume-from", early_create, early_docol_codeword

  ; If the length is zero, exit without doing anything.
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "buffer-logical-length", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 2*8, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; Decrement the logical length. We do this now to get it over with, since
  ; adjusting the start pointer is more complex.
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "buffer-logical-length", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "-", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  ; (metadata pointer)

  ; We compute the physical end. We'll need it in adjusting the logical start,
  ; and doing it now means less stack juggling later.
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "buffer-physical-start", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "buffer-physical-length", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  ; (physical end, metadata pointer)

  ; Compute the incremented logical start.
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "buffer-logical-start", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  ; (physical end, metadata pointer, updated start pointer)

  ; Check whether the updated start is equal to the physical end.
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  ; (metadata pointer, updated start, updated start, physical end)
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma

  ; If the logical start pointer is now equal to the physical end pointer,
  ; we want to wrap to the physical start. That's what makes it a circular
  ; buffer.
  ; (metadata pointer, updated start)
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "buffer-physical-start", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma

  ; However we got here, save the updated logical start pointer.
  ; (metadata pointer, updated start)
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "buffer-logical-start", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store


  ; (metadata pointer)
  dq litstring, "peek-from", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "buffer-logical-length", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 28*8, early_comma

  ; If the length is zero, there is no input, but we can still try calling
  ; the "refill" word.
  ; (metadata pointer)
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "input-buffer-refill", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 17*8, early_comma

  ; If the refill word is nonzero, call it. It expects a copy of the metadata
  ; pointer as its parameter, so set that up.
  ; (metadata pointer, refill word)
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "execute", early_find, entry_to_execution_token, early_comma
  ; (metadata pointer)
  ; Now we check if the length is still zero.
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "buffer-logical-length", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 10*8, early_comma

  ; The length is zero even after calling the refill word, so return null.
  ; (metadata pointer)
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; If the refill word is zero, we can't help, just return null.
  ; (metadata pointer, refill word)
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ;   The buffer is non-empty, so read a byte from it. We might have reached
  ; this point either from the original check, or from the second check after
  ; calling the refill word.
  ;
  ;   While it might be more extensible to call an "unpack" word here, such as
  ; unpack8, we actually just want to get the value, and delegate the pointer
  ; adjustments to "consume".
  ;
  ; (metadata pointer)
  dq litstring, "buffer-logical-start", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "8@", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store


  ; (metadata pointer)
  dq litstring, "key-from", 0, early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "peek-from", early_find, entry_to_execution_token, early_comma
  ; (metadata pointer, result byte)
  ; We unconditionally consume, because we have no way to distinguish between
  ; reading a zero byte and having nothing left to read.
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "consume-from", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store


  dq litstring, "is-space", 0, early_create, early_docol_codeword

  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x20, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x09, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x0a, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x0b, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x0c, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x0d, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store


  dq litstring, "peek", early_create, early_docol_codeword
  dq litstring, "main-input-buffer", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "peek-from", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "consume", early_create, early_docol_codeword
  dq litstring, "main-input-buffer", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "consume-from", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "key", early_create, early_docol_codeword
  dq litstring, "main-input-buffer", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "key-from", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store


  ;;;   As a convenience for "word", we have some facilities for working with
  ;;; stack-allocated strings. Yeah, trippy concept. Also, it would be a
  ;;; buffer overrun hazard if we were worried about that, which is why this
  ;;; is no longer common practice in C.
  ;;;
  ;;;   The most important of these is accumulate-string, but we need some
  ;;; smaller pieces first.

  ; In:
  ;   (multiple words) string
  ;   (multiple words) items to be left alone
  ;   item to be unrolled
  ;   number of items above string that participate in the unroll
  ; Out:
  ;   item that was unrolled
  ;   (multiple words) string
  ;   (multiple words) items left alone
  dq litstring, "unroll-past-string", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "*", early_find, entry_to_execution_token, early_comma
  ; (string, other items, top item, byte offset to string start)
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "value@", early_find, entry_to_execution_token, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  ; We have two copies of the offset present, in addition to the stuff we want
  ; to rotate. So, the actual string starts two words on... We could have
  ; adjusted the offset instead, but we'll want the unmodified offset again
  ; later.
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 16, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  ; (string, other items, top item, offset to start, string pointer)
  dq litstring, "stringlen", early_find, entry_to_execution_token, early_comma
  ; Same reasoning as in accumulate-string (see below).
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "align-size", early_find, entry_to_execution_token, early_comma
  ; (string, other items, top item, offset to start, string length w/ padding)
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "/%", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  ; (string, other items, top item, number of words to unroll)
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; In:
  ;   (multiple words) string
  ;   item to be swapped
  ; Out:
  ;   item that was swapped
  ;   (multiple words) string
  dq litstring, "swap-past-string", 0, early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "unroll-past-string", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; In:
  ;   (multiple words) string
  dq litstring, "dropstring", early_create, early_docol_codeword
  dq litstring, "value@", early_find, entry_to_execution_token, early_comma
  dq litstring, "stringlen", early_find, entry_to_execution_token, early_comma
  ; Same reasoning as in accumulate-string (see below).
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "align-size", early_find, entry_to_execution_token, early_comma
  dq litstring, "value@", early_find, entry_to_execution_token, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  ;   At the time we fetched the stack pointer, there was an extra value atop
  ; it, so we have to add one more word.
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "value!", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; In:
  ;   (multiple words) string
  ;   item to be kept
  ; Out:
  ;   item that was kept
  dq litstring, "dropstring-with-result", early_create, early_docol_codeword
  dq litstring, "swap-past-string", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "dropstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; In:
  ;   (multiple words) string-so-far
  ;   new character byte
  ; Out:
  ;   (multiple words) updated string-so-far
  dq litstring, "accumulate-string", early_create
  dq early_docol_codeword

  ; Compute the address of the final word of the string.
  ;
  ;   It's a little bit difficult to get the start pointer right, since all
  ; our intermediate products affect what we get from value@, so we compute
  ; that just once, here at the beginning.
  dq litstring, "value@", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  ; (string so far, new character byte, pointer to start of string)
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "stringlen", early_find, entry_to_execution_token, early_comma
  ; There are two concerns here that overlap: First, we always want at least
  ; one word. Recall that a length of zero bytes won't receive any alignment
  ; padding because it's already divisible by 8. Second, the result of
  ; stringlen doesn't include the null byte, which might be in a word by
  ; itself that needs to be counted. We can address both of them by
  ; unconditionally adding 1 to the length before applying alignment.
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  ; Pad the length for alignment.
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "align-size", early_find, entry_to_execution_token
  dq early_comma
  ; We want an offset from the first word of the string to the last word of
  ; the string, so we subtract one word from the length.
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "-", early_find, entry_to_execution_token, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  ; (string so far, new character byte, address of final word)

  ; Examine the final word of the string, leaving other stuff undisturbed.
  ; Work low-to-high to figure out where to store the new byte, taking the
  ; first one that's available.
  ; (string so far, new character byte, address of final word)
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x00000000000000FF, early_comma
  dq litstring, "&", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 6*8, early_comma
  ; (string so far, new character byte, address of final word, old value)
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "|", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  ; (string so far, new character byte, address of final word, old value)

  ; This next part is repeated several times, changing only the offsets, for
  ; bytes 1 through 6; bytes 0 (above) and 7 (way below) are different.
  ; (string so far, new character byte, address of final word, old value)
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x000000000000FF00, early_comma
  dq litstring, "&", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 9*8, early_comma
  ; (string so far, new character byte, address of final word, old value)
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x0000000000000100, early_comma
  dq litstring, "*", early_find, entry_to_execution_token, early_comma
  dq litstring, "|", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; Highly repetitive code, see above.
  ; (string so far, new character byte, address of final word, old value)
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x0000000000FF0000, early_comma
  dq litstring, "&", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 9*8, early_comma
  ; (string so far, new character byte, address of final word, old value)
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x0000000000010000, early_comma
  dq litstring, "*", early_find, entry_to_execution_token, early_comma
  dq litstring, "|", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; Highly repetitive code, see above.
  ; (string so far, new character byte, address of final word, old value)
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x00000000FF000000, early_comma
  dq litstring, "&", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 9*8, early_comma
  ; (string so far, new character byte, address of final word, old value)
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x0000000001000000, early_comma
  dq litstring, "*", early_find, entry_to_execution_token, early_comma
  dq litstring, "|", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; Highly repetitive code, see above.
  ; (string so far, new character byte, address of final word, old value)
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x000000FF00000000, early_comma
  dq litstring, "&", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 9*8, early_comma
  ; (string so far, new character byte, address of final word, old value)
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x0000000100000000, early_comma
  dq litstring, "*", early_find, entry_to_execution_token, early_comma
  dq litstring, "|", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; Highly repetitive code, see above.
  ; (string so far, new character byte, address of final word, old value)
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x0000FF0000000000, early_comma
  dq litstring, "&", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 9*8, early_comma
  ; (string so far, new character byte, address of final word, old value)
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x0000010000000000, early_comma
  dq litstring, "*", early_find, entry_to_execution_token, early_comma
  dq litstring, "|", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; Highly repetitive code, see above.
  ; (string so far, new character byte, address of final word, old value)
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x00FF000000000000, early_comma
  dq litstring, "&", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 9*8, early_comma
  ; (string so far, new character byte, address of final word, old value)
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x0001000000000000, early_comma
  dq litstring, "*", early_find, entry_to_execution_token, early_comma
  dq litstring, "|", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ;   The top byte of the final word is always zero (or else stringlen
  ; wouldn't have called it the final word), so we don't need to check it, we
  ; can just use it.
  ;
  ;   We need to put the new value in the top byte, which will mean we have no
  ; null terminator, so we also need to start a new word.
  ;
  ;   There is a fiddly order-dependency here: unroll-past-string relies on
  ; being able to find the null terminator, which won't work if we've gotten
  ; rid of it. Also, calling it will move all the earlier words, including
  ; the one we intend to write to, which will invalidate any pointer we're
  ; keeping at that point. There's a few ways to resolve this; what we do is
  ; put the new terminator in place first, manually nudge the pointer, and
  ; then write the new value.
  ; (string so far, new character byte, address of final word, old value)
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x0100000000000000, early_comma
  dq litstring, "*", early_find, entry_to_execution_token, early_comma
  dq litstring, "|", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  ; (string so far, new value, address of final word)
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 3, early_comma
  dq litstring, "unroll-past-string", early_find, entry_to_execution_token
  dq early_comma
  ; (new null terminator, string so far, new value, invalid address)
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "-", early_find, entry_to_execution_token, early_comma
  ; (new null terminator, string so far, new value, updated address)
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  ; (updated string)
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store


  ; Okay, this is the big one, the lexer! Wow.
  ;
  ; Out:
  ;   (multiple words) string
  dq litstring, "word", early_create, early_docol_codeword

  ; We allocate an empty string first, so that the result of "key" will
  ; conveniently be on the easy-to-find end of it.
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma

  ; Skip whitespace.
  dq litstring, "key", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "is-space", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 4*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "branch", early_find, entry_to_execution_token, early_comma
  dq lit, -7*8, early_comma

  ; Early exit if it's a zero byte.
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 6*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "dropstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  dq litstring, "accumulate-string", early_find, entry_to_execution_token
  dq early_comma

  dq litstring, "peek", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "is-space", 0, early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 8*8, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma
  dq litstring, "consume", early_find, entry_to_execution_token, early_comma
  dq litstring, "accumulate-string", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "branch", early_find, entry_to_execution_token, early_comma
  dq lit, -14*8, early_comma

  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store


  dq litstring, "find", early_create, early_docol_codeword
  dq litstring, "latest", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "find-in", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store


  ; In:
  ;   character
  ; Out:
  ;   1 for true, 0 for false
  dq litstring, "is-alphanumeric", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, "0", early_comma
  dq litstring, ">", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma

  ; Less than "0".
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, "9", early_comma
  dq litstring, ">=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma

  ; Less than or equal to "9".
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, "A", early_comma
  dq litstring, ">", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma

  ; Less than "A".
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, "Z", early_comma
  dq litstring, ">=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma

  ; Less than or equal to "Z".
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, "a", early_comma
  dq litstring, ">", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma

  ; Less than "a".
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, "z", early_comma
  dq litstring, ">=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 5*8, early_comma

  ; Less than or equal to "z".
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; Greater than "z".
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store


  ; In:
  ;   character
  ; Out:
  ;   value
  dq litstring, "generalized-digit-value", early_create, early_docol_codeword

  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, "a", early_comma
  dq litstring, "<=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 8*8, early_comma

  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, "a", early_comma
  dq litstring, "-", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 10, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, "A", early_comma
  dq litstring, "<=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 8*8, early_comma

  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, "A", early_comma
  dq litstring, "-", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 10, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, "0", early_comma
  dq litstring, "-", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store


  ; In:
  ;   character
  ;   base
  ; Out:
  ;   value (if successful)
  ;   error indicator (zero indicates success)
  dq litstring, "decode-generalized-digit", 0, early_create
  dq early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "is-alphanumeric", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 14*8, early_comma

  ; It's alphanumeric.
  ; (base, character)
  dq litstring, "generalized-digit-value", early_find
  dq entry_to_execution_token, early_comma
  ; (base, value)
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  ; (value, value, base)
  dq litstring, ">", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 4*8, early_comma

  ; It's in range.
  ; (value)
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; It's out of range.
  ; (value)
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; It's not alphanumeric.
  ; (base, character)
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store


  ; In:
  ;   string pointer
  ;   base
  ; Out:
  ;   result (if successful)
  ;   error indicator (zero indicates success)
  dq litstring, "read-base-unsigned", early_create, early_docol_codeword
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma

  ; If the first byte is null, this is an error
  dq litstring, "unpack8", early_find, entry_to_execution_token, early_comma
  ; (numeric base, current point in string, character)
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 7*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; Decode the first byte as a generalized digit in the base.
  ; (numeric base, current point in string, character)
  ; If the first byte is less than "0", this is an error.
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  ; (numeric base, current point in string, character, numeric base)
  dq litstring, "decode-generalized-digit", 0, early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 6*8, early_comma

  ; (numeric base, current point in string)
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; The first byte is a valid generalized digit in the appropriate base, so
  ; let's get started.
  ; (numeric base, current point in string, initial value)
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma

  ; Start of the loop.
  ; (numeric base, result so far, current point in string)
  dq litstring, "unpack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 8*8, early_comma

  ; A null after the first character is valid, and indicates we're done.
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; Decode the latest byte as a generalized digit in the base.
  ; (numeric base, result so far, current point in string, latest byte)
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  ; (numeric base, result so far, current point in string, character
  ;  numeric base)
  dq litstring, "decode-generalized-digit", 0, early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 7*8, early_comma

  ; If the latest character is not a valid digit, that's an error.
  ; (numeric base, result so far, current point in string)
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; The latest character is valid, so incorporate it and loop.
  ; (numeric base, result so far, current point in string, latest value)
  dq litstring, "3roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 4, early_comma
  dq litstring, "roll", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 5, early_comma
  dq litstring, "unroll", early_find, entry_to_execution_token, early_comma
  dq litstring, "*", early_find, entry_to_execution_token, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "branch", early_find, entry_to_execution_token, early_comma
  dq lit, -42*8, early_comma

  dq early_here, fetch, lit, 8, packalign, early_here_store


  ; In:
  ;   string pointer
  ; Out:
  ;   result (if successful)
  ;   error indicator (zero indicates success)
  dq litstring, "read-integer-unsigned", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "unpack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, "0", early_comma
  dq litstring, "!=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 6*8, early_comma

  ; This is the case where the leading digit is not a zero.
  ; (original string pointer, advanced string pointer)
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 10, early_comma
  dq litstring, "read-base-unsigned", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; This is the case where the leading digit is a zero.
  ; (original string pointer, advanced string pointer)
  dq litstring, "unpack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, "x", early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 8*8, early_comma

  ; This is the case where the second character is equal to "x".
  ; (original string pointer, doubly advanced string pointer, character)
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 16, early_comma
  dq litstring, "read-base-unsigned", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, "o", early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 8*8, early_comma

  ; This is the case where the second character is equal to "o".
  ; (original string pointer, doubly advanced string pointer, character)
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "read-base-unsigned", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, "b", early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 8*8, early_comma

  ; This is the case where the second character is equal to "b".
  ; (original string pointer, doubly advanced string pointer, character)
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 2, early_comma
  dq litstring, "read-base-unsigned", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; This is the case where the second character is something else.
  ; (original string pointer, doubly advanced string pointer, character)
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 10, early_comma
  dq litstring, "read-base-unsigned", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma


  ; In:
  ;   string pointer
  ; Out:
  ;   result (if successful)
  ;   error indicator (zero indicates success)
  dq litstring, "read-integer", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "unpack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, "-", early_comma
  dq litstring, "!=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 4*8, early_comma

  ; This is the case where it's non-negative.
  ; (original string pointer, advanced string pointer)
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "read-integer-unsigned", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; This is the case where it's negative.
  ; (original string pointer, advanced string pointer)
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "read-integer-unsigned", early_find, entry_to_execution_token
  dq early_comma
  ; (result maybe, exit code)
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 2*8, early_comma

  ; Failure
  ; (non-zero exit code)
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; Success
  ; (result, zero exit code)
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, -1, early_comma
  dq litstring, "*", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store


  ; In:
  ;   string pointer
  ; Out:
  ;   result (if successful)
  ;   error indicator (zero indicates success)
  dq litstring, "read-decimal", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "unpack8", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, "-", early_comma
  dq litstring, "!=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 6*8, early_comma

  ; This is the case where it's non-negative.
  ; (original string pointer, advanced string pointer)
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 10, early_comma
  dq litstring, "read-base-unsigned", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; This is the case where it's negative.
  ; (original string pointer, advanced string pointer)
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 10, early_comma
  dq litstring, "read-base-unsigned", early_find, entry_to_execution_token
  dq early_comma
  ; (result maybe, exit code)
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 2*8, early_comma

  ; Failure
  ; (non-zero exit code)
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; Success
  ; (result, zero exit code)
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, -1, early_comma
  dq litstring, "*", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store


  dq litstring, "interpreter-flags-storage", early_create
  dq litstring, "interpreter-flags-storage", early_find
  dq dup, lit, 1, store_entry_flags
  dq entry_to_execution_token
  ; (heap pointer, storage pointer)
  dq swap, lit, 0, early_comma
  dq litstring, "variable", 0, early_find, entry_to_execution_token
  ; (storage pointer, heap pointer, execution token)
  dq swap, unroll3
  ; (heap pointer, storage pointer, execution token)
  dq litstring, "interpreter-flags", swap
  ; (heap pointer, storage pointer, string pointer, execution token)
  dq execute


  dq litstring, "hide-entry", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "entry-flags@", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x80, early_comma
  dq litstring, "|", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "entry-flags!", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store


  dq litstring, "unhide-entry", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "entry-flags@", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x80, early_comma
  dq litstring, "invert", early_find, entry_to_execution_token, early_comma
  dq litstring, "&", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "entry-flags!", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store


  dq litstring, "set-word-immediate", early_create, early_docol_codeword
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "entry-flags@", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x01, early_comma
  dq litstring, "|", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "entry-flags!", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store


  dq litstring, "[", early_create, early_docol_codeword
  dq litstring, "interpreter-flags", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x01, early_comma
  dq litstring, "invert", early_find, entry_to_execution_token, early_comma
  dq litstring, "&", early_find, entry_to_execution_token, early_comma
  dq litstring, "interpreter-flags", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; Run some imperative code to fix the flags.
  dq litstring, "set-word-immediate", early_find, entry_to_execution_token
  dq swap, litstring, "[", early_find, roll3, execute


  dq litstring, "]", early_create, early_docol_codeword
  dq litstring, "interpreter-flags", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x01, early_comma
  dq litstring, "|", early_find, entry_to_execution_token, early_comma
  dq litstring, "interpreter-flags", early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store


  dq litstring, ":", early_create, early_docol_codeword
  dq litstring, "word", early_find, entry_to_execution_token, early_comma
  dq litstring, "value@", early_find, entry_to_execution_token, early_comma
  dq litstring, "create", early_find, entry_to_execution_token, early_comma
  dq litstring, "dropstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "docol", early_find, entry_to_execution_token, early_comma
  dq litstring, ",", early_find, entry_to_execution_token, early_comma
  dq litstring, "latest", early_find, entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "hide-entry", early_find, entry_to_execution_token
  dq early_comma
  ; If this feels backwards, imagine to yourself that everything that ISN'T
  ; defining a word body is part of an implicit [ ... ] sequence. Doing so
  ; doesn't really change anything, but may make you happier.
  dq litstring, "]", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store


  dq litstring, ";", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq litstring, ",", early_find, entry_to_execution_token, early_comma
  dq litstring, "latest", early_find, entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "unhide-entry", early_find, entry_to_execution_token
  dq early_comma
  ; As above.
  dq litstring, "[", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  ; Run some imperative code to fix the flags.
  dq litstring, "set-word-immediate", early_find, entry_to_execution_token
  dq swap, litstring, ";", early_find, roll3, execute


  dq litstring, ";asm", early_create, early_docol_codeword
  dq litstring, "here", early_find, entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "pack-next", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "packalign", early_find, entry_to_execution_token, early_comma
  dq litstring, "here", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "latest", early_find, entry_to_execution_token, early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "unhide-entry", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "entry-to-execution-token", 0, early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 8, early_comma
  dq litstring, "+", early_find, entry_to_execution_token, early_comma
  dq litstring, "swap", early_find, entry_to_execution_token, early_comma
  dq litstring, "!", early_find, entry_to_execution_token, early_comma
  dq litstring, "[", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store

  dq litstring, "set-word-immediate", early_find, entry_to_execution_token
  dq swap, litstring, ";asm", early_find, roll3, execute


  ; Although we will eventually define the word "'" to give us the symbol of a
  ; word, it will rely on being able to compile a literal. Rather than do
  ; lots of string processing later, we choose to define this word now to
  ; avoid having to look up the word "lit" as part of that.
  ;
  ; In:
  ;   value
  dq litstring, "literal", early_create, early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq litstring, ",", early_find, entry_to_execution_token, early_comma
  dq litstring, ",", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store


  ; Now the single most important word...
  dq litstring, "interpret", early_create, early_docol_codeword
  dq litstring, "word", early_find, entry_to_execution_token, early_comma

  ; If no word was returned, exit.
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 3*8, early_comma
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; The string is on the top of the stack, so to get a pointer to it we get
  ; the stack address.
  ; (string)
  dq litstring, "value@", early_find, entry_to_execution_token, early_comma
  dq litstring, "find", early_find, entry_to_execution_token, early_comma

  ; Check whether the word was found in the dictionary.
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "!=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 25*8, early_comma

  ; If the word is in the dictionary, check what mode we're in, then...
  dq litstring, "dropstring-with-result", early_find, entry_to_execution_token
  dq early_comma
  ; (entry pointer)
  dq litstring, "interpreter-flags", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x01, early_comma
  dq litstring, "&", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 14*8, early_comma

  ; ... if we're in compile mode, there's still a chance it's an immediate
  ; word, in which case we branch over to interpret mode...
  dq litstring, "dup", early_find, entry_to_execution_token, early_comma
  dq litstring, "entry-flags@", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 1, early_comma
  dq litstring, "&", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 4*8, early_comma
  ; ... but it's a regular word, so append it to the heap.
  dq litstring, "entry-to-execution-token", 0, early_find
  dq entry_to_execution_token, early_comma
  dq litstring, ",", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; ... if we're in interpret mode, or the word is immediate, run it.
  dq litstring, "entry-to-execution-token", 0, early_find
  dq entry_to_execution_token, early_comma
  dq litstring, "execute", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; If it's not in the dictionary, check whether it's a decimal number.
  dq litstring, "drop", early_find, entry_to_execution_token, early_comma
  ; As before, we get the stack address and use it as a string pointer.
  ; (string)
  dq litstring, "value@", early_find, entry_to_execution_token, early_comma
  dq litstring, "read-integer", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0, early_comma
  dq litstring, "=", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 16*8, early_comma

  ; It's a number.
  dq litstring, "interpreter-flags", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "@", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq lit, 0x01, early_comma
  dq litstring, "&", early_find, entry_to_execution_token, early_comma
  dq litstring, "0branch", early_find, entry_to_execution_token, early_comma
  dq lit, 7*8, early_comma

  ; We're in compile mode; append first "lit", then the number, to the heap.
  ; The version of "lit" we use is the one that's current when we ourselves
  ; are compiled, hardcoded; doing a dynamic lookup would require dealing with
  ; what happens if it's not found.
  dq litstring, "dropstring-with-result", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq litstring, ",", early_find, entry_to_execution_token, early_comma
  dq litstring, ",", early_find, entry_to_execution_token, early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; We're in interpret mode; push the number to the stack. Or at least, that's
  ; what the code we're interpreting will see. Really it's already on the
  ; stack, just clean everything else up and leave it there.
  dq litstring, "dropstring-with-result", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma

  ; If it's neither in the dictionary nor a number, just print an error.
  dq litstring, "litstring", early_find, entry_to_execution_token, early_comma
  dq lit, "No such ", early_comma, lit, "word: ", early_comma
  dq litstring, "emitstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "value@", early_find, entry_to_execution_token, early_comma
  dq litstring, "emitstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "dropstring", early_find, entry_to_execution_token
  dq early_comma
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store


  ;   The next layer is built now, so let's move on to it.
  ;
  ;   We've chosen to defer creating "quit" until we're in interpreted mode,
  ; so we can't call "quit" here, we need to call "interpret". Since
  ; "interpret" doesn't loop, we need to call it a few times in a row - just
  ; enough times for the code in boot_source, which it'll be interpreting, to
  ; create and invoke "quit".
  ;
  ;   We could allocate a real word that does that, but then it would stick
  ; around after it's no longer needed. Instead, we create a trampoline, which
  ; has a "docol" codeword, some calls to "interpret", but nothing else, then
  ; we call it the same way we would call an ordinary word given its execution
  ; token.
  ;
  ;   To avoid needing to explicitly clean up the trampoline, we never
  ; formally allocate it. Since we know exactly what code it'll be running, we
  ; also know how much heap space that code will use. We put the trampoline
  ; immediately after the end of where "quit" will be, so that the very next
  ; heap allocations after creating "quit" will overwrite the trampoline.
  ;
  ;   For simplicity's sake, we use the flatassembler copies of docol, lit,
  ; and sys_exit. It would be possible to use the heap copies, but we have
  ; these ones for just a few moments longer, so we might as well use them.
  dq litstring, "interpret", early_find, entry_to_execution_token
  dq swap, early_here, fetch
  ; (interpret, heap pointer, here pointer)

  ;   Consult the definition of "quit" in boot_source and count how many words
  ; the compiled implementation of it will take up.
  ;
  ;   That will include one for the next-entry pointer; one for the name and
  ; flags (it's short enough to fit, even with the null terminators); one for
  ; the codeword pointer to docol; one for each word that's explicitly added
  ; as part of its body in the source; and one for a trailing "exit" that's
  ; added by ";" and never reached.
  ;
  ;   Whatever total you get, that's how many words to add to "here", on this
  ; line, to obtain the trampoline's start location.
  ;
  ;   Notionally, less might appear to work, but you'd be racing to execute a
  ; word before overwriting it, so it isn't a great idea.
  dq lit, 10*8, add
  ; (interpret, heap pointer, start of trampoline)

  dq dup, lit, 4, roll, swap
  ; (heap pointer, start of trampoline, interpret, current point in trampoline)
  dq lit, docol, pack64

  ;   In boot_source, count how many interpreted words there are from the
  ; initial ":" to the invocation of "quit", inclusive. Don't forget that
  ; ": quit" is only a single interpreted word, because ":" reads a word to
  ; use as a name, and that one is never seen by "interpret".
  ;
  ;   Whatever total you get, adjust the number of repeated lines just below,
  ; to put that exact number of copies of the execution token for "interpret"
  ; into the trampoline.
  ;
  ; (heap pointer, start of trampoline, interpret, current point in trampoline)
  dq swap, dup, unroll3, pack64
  dq swap, dup, unroll3, pack64
  dq swap, dup, unroll3, pack64
  dq swap, dup, unroll3, pack64
  dq swap, dup, unroll3, pack64
  dq swap, dup, unroll3, pack64
  dq swap, dup, unroll3, pack64
  dq swap, dup, unroll3, pack64
  dq swap, dup, unroll3, pack64
  dq swap, dup, unroll3, pack64
  dq swap, dup, unroll3, pack64
  dq swap, dup, unroll3, pack64
  dq swap, dup, unroll3, pack64
  dq swap, dup, unroll3, pack64

  dq swap, drop
  ; (heap pointer, start of trampoline, current point in trampoline)

  ; The trampoline will never get past the above calls to "interpret", but if
  ; it does, complain.
  dq lit, lit, pack64, lit, 1, pack64, lit, sys_exit, pack64
  dq drop
  ; (heap pointer, start of trampoline)

  ; Get rid of that heap pointer on the stack, we're finally done with it!
  dq swap, drop
  dq execute

  ; We won't be coming back, but if we do, complain about that, too.
  dq lit, 1, sys_exit


  ;;; For triage's sake, here's an inventory of everything else in the file.
  ;;;
  ;;; Macros:
  ;;;   next, beforenext, pushcontrol, popcontrol
  ;;; Assembly:
  ;;;   docol (the constant, and the actual codeword implementation)
  ;;;     needs a label
  ;;;     pushcontrol, next
  ;;;   exit
  ;;;     popcontrol, next
  ;;;   swap, drop, drop2, roll, unroll, roll3, unroll3, dup, dup2
  ;;;   add, sub, mul, divmod
  ;;;   eq, ne, gt, lt, ge, le
  ;;;   gt_unsigned, lt_unsigned, ge_unsigned, le_unsigned
  ;;;   and, or, xor, invert
  ;;;   lit, litstring
  ;;;   store, fetch
  ;;;   store_control_stack,fetch_control_stack,
  ;;;   store_value_stack, fetch_value_stack,
  ;;;   addstore, substore, store8, fetch8, store16, fetch16, store32, fetch32
  ;;;   memcopy, stringlen
  ;;;   reverse_stringlen, reverse_padding_len
  ;;;   stringcmp
  ;;;     no dependencies except next for any of these
  ;;;   branch, zbranch
  ;;;     sorta needs a label but might be avoidable
  ;;;     needs next
  ;;;   execute
  ;;;   sys_exit, sys_write
  ;;;     nothing special
  ;;; Forth:
  ;;;   emitstring
  ;;;     stringlen, sys_write, basics
  ;;;   pow logfloor logceil
  ;;;   dotbase_unsigned, dotbase, dot
  ;;;   dothex, dothex8, dothex16, dothex32, dothex64
  ;;; Assembly:
  ;;;   crash
  ;;;     nothing special
  ;;; Forth:
  ;;;   pack64, pack32, pack16, pack8, packstring, packalign
  ;;;     only the basics above
  ;;; Assembly:
  ;;;   litpack64, litpack32, litpack16, litpack8
  ;;;     beforenext, pack*
  ;;; Forth:
  ;;;   rax, rcx, rdx, rbx, rsp, rbp, rsi, rdi
  ;;;   r8, r9, r10, r11, r12, r13, r14, r15
  ;;;   eax, ecx, edx, ebx, esp, ebp, esi, edi
  ;;;   ax, cx, dx, bx, sp, bp, si, di
  ;;;   al, cl, dl, bl, ah, ch, dh, bh
  ;;;   cc_overflow, cc_no_overflow, cc_below, cc_above_equal, cc_equal,
  ;;;   cc_not_equal, cc_below_equal, cc_above, cc_sign, cc_not_sign, cc_even,
  ;;;   cc_odd, cc_less, cc_greater_equal, cc_less_equal, cc_greater
  ;;;     only the basics above
  ;;;   reg64, extrareg64, reg32, reg16, reg8, scalefield, conditioncode
  ;;;     only the basics plus optional emitstring and sys_exit
  ;;;     (notice that these are forward references!)
  ;;;   rex_w, rex_wb, opcodereg, opcodecc, modrm, sib
  ;;;     only the basics above
  ;;;   addressing_reg64, addressing_indirect_reg64, addressing_disp8_reg64,
  ;;;   addressing_indexed_reg64, addressing_disp8_indexed_reg64,
  ;;;   addressing_reg8
  ;;;     basics plus earlier assembly stuff
  ;;;   cld, std,
  ;;;   mov_reg64_imm32,
  ;;;   mov_reg64_imm64, mov_extrareg64_imm64,
  ;;;   mov_reg64_reg64,
  ;;;   mov_indirect_reg64_reg64, mov_disp8_reg64_reg64,
  ;;;   mov_reg64_indirect_reg64, mov_reg64_disp8_reg64,
  ;;;   mov_reg64_indexed_reg64, mov_indexed_reg64_reg64,
  ;;;   mov_indirect_reg64_reg32, mov_disp8_reg64_reg32,
  ;;;   mov_reg32_indirect_reg64, mov_reg32_disp8_reg64,
  ;;;   mov_indirect_reg64_reg16, mov_disp8_reg64_reg16,
  ;;;   mov_reg16_indirect_reg64, mov_reg16_disp8_reg64,
  ;;;   mov_indirect_reg64_reg8, mov_disp8_reg64_reg8,
  ;;;   mov_reg8_indirect_reg64, mov_reg8_disp8_reg64,
  ;;;   mov_reg8_reg8,
  ;;;   lea_reg64_disp8_reg64, lea_reg64_indexed_reg64,
  ;;;   lea_reg64_disp8_indexed_reg64,
  ;;;   push_reg64, pop_reg64, push_imm32_extended,
  ;;;   movs8, movs16, movs32, movs64,
  ;;;   rep_movs8, rep_movs16, rep_movs32, rep_movs64,
  ;;;   lods8, lods16, lods32, lods64,
  ;;;   rep_lods8, rep_lods16, rep_lods32, rep_lods64,
  ;;;   stos8, stos16, stos32, stos64,
  ;;;   rep_stos8, rep_stos16, rep_stos32, rep_stos64,
  ;;;   cmps8, cmps16, cmps32, cmps64,
  ;;;   repz_cmps8, repz_cmps16, repz_cmps32, repz_cmps64,
  ;;;   repnz_cmps8, repnz_cmps16, repnz_cmps32, repnz_cmps64,
  ;;;   scas8, scas16, scas32, scas64,
  ;;;   repz_scas8, repz_scas16, repz_scas32, repz_scas64,
  ;;;   repnz_scas8, repnz_scas16, repnz_scas32, repnz_scas64,
  ;;;   add_reg64_reg64, add_indirect_reg64_reg64, add_reg64_indirect_reg64,
  ;;;   add_reg64_imm8,
  ;;;   sub_reg64_reg64, sub_indirect_reg64_reg64,
  ;;;   sub_reg64_imm8, sbb_reg64_imm8
  ;;;   mul_reg64, divmod_reg64, idivmod_reg64, inc_reg64, dec_reg64,
  ;;;   and_reg64_reg64, and_reg64_imm8,
  ;;;   or_reg64_reg64, or_reg64_imm8,
  ;;;   xox_reg64_reg64,
  ;;;   not_reg64,
  ;;;   cmp_reg64_reg64, test_reg64_reg64,
  ;;;   set_reg8_cc, jmp_cc_rel_imm8, jmp_cc_rel_im32,
  ;;;   jmp_abs_indirect_reg64, jmp_rel_imm8, jmp_rel_imm32,
  ;;;   syscall, hlt
  ;;;     basics plus assembly helpers
  ;;;   pack_next, pack_beforenext, pack_pushcontrol, pack_popcontrol,
  ;;;     basics, assembly stuff
  ;;;   entry_to_execution_token, execution_token_to_entry,
  ;;;   fetch_entry_flags, store_entry_flags,
  ;;;   entry_to_name,
  ;;;   find_in, next_newer_entry_in, guess_entry_end_in,
  ;;;   show_hex_between, show_source_between, show_source_or_hex_between,
  ;;;     some interdependencies here but it's all topologically sorted
  ;;; Forth, not needed on heap:
  ;;;   early_heap, early_s0, early_r0, early_latest, early_here
  ;;;   early_find, early_next_newer_entry, early_guess_entry_end,
  ;;;   early_show_hex, early_show_source, early_show_source_or_hex
  ;;;   early_describe, early_describe_all,
  ;;;     TODO there should really be non-early versions of these two
  ;;;   early_create, early_comma, early_self_codeword, early_docol_codeword,
  ;;;   early_here_store, early_variable
  ;;;
  ;;; It's likely that nothing past this point is required for the heap copy,
  ;;; but it's here for completeness.
  ;;;
  ;;; Forth, subject to reconsideration:
  ;;;   quit
  ;;;     quine (this is a forward reference)
  ;;; Forth:
  ;;;   quine
  ;;;     early_here (removable), all_contents (forward)
  ;;;     sys_write, basics
  ;;;   use_label, set_label
  ;;;     only basics
  ;;; Forth:
  ;;;   all_contents, elf_file_header, elf_program_header
  ;;;   output_start_routine
  ;;;     nothing special in any of these (they do want use/set_label)
  ;;;   self_raw
  ;;;     self-reference
  ;;; Not a real word:
  ;;;   end_segment
  ;;;
  dq quit

;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; Now we are in Forth ;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;
;;;   Everything we define from here on out is an actual Forth word, with a
;;; proper header and everything. So, you'll see some more preamble before the
;;; definitions.
;;;
;;;   Keep in mind, though, that, although we have threaded execution, we
;;; don't yet have Forth-style variables. That's because the heap is at a
;;; dynamically-chosen location, so none of this read-only code that we're
;;; defining now can reference it. Before invoking cold_start, we thoughtfully
;;; put the value of "heap" on the stack for ourselves; our first task will be
;;; to dynamically allocate some words on the heap that know how to find the
;;; heap. We'll do that by defining bootstrapping versions of the
;;; word-defining words, which will eventually be replaced.

; Token pasting is possible in flatassembler, but kind of a pain.
calminstruction defword_label_helper name
  local full_name, address
  arrange full_name, name#=_name        ; do the token-pasting
  compute address, $                    ; compute the current address
  publish full_name:, address           ; bind the label
end calminstruction

; The only time we actually use this variant is docol.
macro defword_unlabeled name, flags
  align 8
  defword_label_helper name
  dq latest_word
  latest_word = $ - 8
  db flags, 0x00, `name, 0x00
  align 8
end macro

; This is the variant we use to define ordinary words.
macro defword name, flags
  defword_unlabeled name, flags
  label name
end macro

latest_word = 0

;;;
;;; Routine docol
;;; -------------
;;;
;;;   Reference this via its label as the codeword of a word to make it an
;;; "interpreted" word. Concretely, it saves rsi (the "instruction pointer")
;;; to the control stack, takes the address of the codeword from rax and
;;; increments it in-place to form the new instruction pointer, and copies
;;; that to rsi.
;;;
;;;   Having then done this, we're now in the state that normal execution
;;; expects, so docol ends by it using "next" to begin the callee's execution,
;;; kicking off a nested call.
;;;
;;;   The name is said to be short for "do colon", because Forth high-level
;;; code begins word definitions with a colon.
;;;
;;; Registers in:
;;;
;;; * rsi is the caller's instruction pointer
;;; * rbp is the control stack pointer
;;; * rax is the address of the callee's codeword
;;;
;;; Registers out:
;;;
;;; * rsi is the callee's instruction pointer
;;; * rbp is the control stack pointer
defword_unlabeled docol, 0
docol_constant:
  ; Evaluated as a word, docol is a constant which returns a pointer.
  dq $ + 8
  mov.qreg.qimm rax, docol
  push.qreg rax
  next
  align 8
docol:
  ; Since docol is not a normal word, the label points to the value we care
  ; about from the assembly side of things, which is the address we use as the
  ; codeword.
  pushcontrol rsi
  add.qreg.bimm rax, 8
  mov.qreg.qreg rsi, rax
  next

latest_word = docol_name

;;;
;;;   This is the mechanism to "return" from a word interpreted by docol.
;;; We pop the control stack, and then, since this is threaded execution, we
;;; do the next thing the caller wants to do, by inlining "next".
;;;
defword exit, 0
  dq $ + 8
  popcontrol rsi
  next


;;;
;;; Stack manipulation routines
;;; ---------------------------
;;;
;;;   We start with the three traditional stack operations, swap drop and
;;; roll. Sorry to fans of the name "ROT"; we were an HP calculator kid. It'll
;;; always be roll to us. Anyway, we do a couple other operations too. Since
;;; our goal right now is just to bootstrap the heap, we keep this short and
;;; sweet.
;;;
;;;   There is definitely plenty of optimization that could be done.
;;;

defword swap, 0
  dq $ + 8
  pop.qreg rax
  pop.qreg rbx
  push.qreg rax
  push.qreg rbx
  next

defword drop, 0
  dq $ + 8
  pop.qreg rax
  next

defword drop2, 0
  dq $ + 8
  pop.qreg rax
  pop.qreg rax
  next

; Rotates "up" (pops its parameter, n; nth item then becomes current item).
;
; We implement this the high-performance way, with rep movsq, aka the
; instruction that exists to optimize C's memcpy(). The details of setting
; that up are complex; see below.
defword roll, 0
  dq $ + 8

  ;   Pop our parameter. The rep instruction takes rcx as its count, so we
  ; reduce copying by using it to hold our count, as well.
  pop.qreg rcx

  ;   We have n - 1 items to slide, so decrement rcx. For the purpose of
  ; counting how many repetitions will happen, it's one-based. This is because
  ; the rep instruction performs a single movsq, then decrements rcx, then
  ; stops if rcx is zero.
  dec.qreg rcx

  ;   Retrieve the nth item, for later. For this purpose we're thinking in
  ;  zero-based terms, so we do this after already having decremented rcx.
  mov.qreg.indexed.qreg rbx, rsp, rcx, 8

  ;   The source address for movsq is rsi and the destination is rdi; we can
  ; use rdi as we wish, but rsi is our Forth "instruction pointer", so we must
  ; save and restore it. Doing so alters rsp, so we have to adjust the address
  ; calculations by eight bytes as compared to the expressions above, but
  ; happily we can use the disp8 field to do that. We'd be using disp8 anyway
  ; because it's helpful.
  push.qreg rsi

  ;   Now we set up parameters for the memory-sliding operation. We have
  ; n - 1 items to copy, moving the range rsp through rsp + (n-2)*8 onto the
  ; range rsp + 8 through rsp + (n-1)*8. That's with the value of rsp as it
  ; exists at this moment (it's going to change soon).
  ;
  ;   We're sliding them upwards in memory, so we start at the high end so
  ; that we're always moving into a location that doesn't have anything
  ; precious. We use lea as a convenient way to do the stack math.
  ;
  ;   When rcx is 1, we want rsp + 8.
  lea.qreg.indexed.qreg rsi, rsp, rcx, 8
  ;   When rcx is 1, we want rsp + 16.
  lea.qreg.disp8.indexed.qreg rdi, 8, rsp, rcx, 8
  ;
  ;   Using rcx = 1 is the most convenient example to use for figuring out the
  ; arithmetic. It's a linear relationship, so as long as we get the 8-byte
  ; stride correct, we just need to pick a single point and verify that our
  ; math is right for that point, and it'll be right for any value of rcx.

  ;   Another of our Forth conventions is that the DF flag should be kept at
  ; zero, which directs string instruction to increment rsi. Here, however,
  ; because our source and destination ranges overlap, we need to start at the
  ; high end, which means we need it to decrement. So we set DF to one, and
  ; we'll clear it after.
  std
  rep movsq

  ; Set everything back.
  cld
  pop.qreg rsi

  ; There is now an extra item at the low end of the stack (the top) that
  ; needs to go away, and coincidentally we have a value in rbx that needs to
  ; be in that spot. Rather than doing a drop and push, we overwrite it, to
  ; save a little work.
  mov.indirect.qreg.qreg rsp, rbx

  ; All done, wow! What a mouthful.
  next

; Rotates "down" (pops its parameter, n; current item then becomes nth item).
;
; We implement this the high-performance way, with rep movsq, aka the
; instruction that exists to optimize C's memcpy(). The details of setting
; that up are complex; see below.
defword unroll, 0
  dq $ + 8

  ;   Pop our parameter. The rep instruction takes rcx as its count, so we
  ; reduce copying by using it to hold our count, as well.
  pop.qreg rcx

  ; We have n - 1 items to slide, so decrement rcx. Also, save a copy of it in
  ; rdx after doing that, for later.
  dec.qreg rcx
  mov.qreg.qreg rdx, rcx

  ;   Retrieve the 0th item, for later.
  mov.qreg.indirect.qreg rbx, rsp

  ;   Now we set up parameters for the memory-sliding operation. We have
  ; n - 1 items to copy, moving the range rsp + 8 through rsp + (n-1)*8 onto
  ; the range rsp through rsp + (n-2)*8. That's with the value of rsp as it
  ; exists at this moment (it's going to change soon).
  ;
  ;   We're sliding them downwards in memory, so we start at the low end so
  ; that we're always moving into a location that doesn't have anything
  ; precious. We use lea as a convenient way to do the stack math.
  ;
  ;   As with roll, we need to save rsi and adjust those rsp calculations
  ; accordingly.
  push.qreg rsi

  ;   Regardless of rcx, we want rsp + 16.
  lea.qreg.disp8.qreg rsi, 16, rsp
  ;   Regardless of rcx, we want rsp + 8.
  lea.qreg.disp8.qreg rdi, 8, rsp

  ;   With roll, we were starting at the high end. Here, we start at the low
  ; end, which means we need rsi to increment after each repetition. That's
  ; what it does when the DF flag is clear, and another of our Forth
  ; conventions is to keep it clear normally. So, we don't have to touch DF!
  ; Yay!
  rep movsq

  ; Restore our original rsi.
  pop.qreg rsi

  ; There is now an extra item in the middle of the stack, at the high end of
  ; the sliding we did, that needs to be overwritten with our value in rbx.
  ; Since we destructively updated our count in rcx, we saved a copy of the
  ; count in rdx, and we use that to find the right address.
  ;
  ; When the original count was n, we want rsp + (n-1)*8, so we saved rdx
  ; after decrementing rcx, above.
  mov.indexed.qreg.qreg rsp, rdx, 8, rbx

  ; All done, wow! What a mouthful.
  next

; Rotates "up" (third item becomes current item)
defword roll3, 0
  dq $ + 8
  pop.qreg rax
  pop.qreg rbx
  pop.qreg rcx
  push.qreg rbx
  push.qreg rax
  push.qreg rcx
  next

; Rotates "down" (current item becomes third item)
defword unroll3, 0
  dq $ + 8
  pop.qreg rax
  pop.qreg rbx
  pop.qreg rcx
  push.qreg rax
  push.qreg rcx
  push.qreg rbx
  next

defword dup, 0
  dq $ + 8
  pop.qreg rax
  push.qreg rax
  push.qreg rax
  next

defword dup2, 0
  dq $ + 8
  pop.qreg rax
  pop.qreg rbx
  push.qreg rbx
  push.qreg rax
  push.qreg rbx
  push.qreg rax
  next

;;;
;;; Arithmetic routines
;;; -------------------
;;;
;;;   No surprises here. Again, since our goal is to bootstrap the heap, we
;;; keep it short. Also again, this is nowhere near optimal.
;;;

defword add, 0
  dq $ + 8
  pop.qreg rbx
  pop.qreg rax
  add.qreg.qreg rax, rbx
  push.qreg rax
  next

defword sub, 0
  dq $ + 8
  pop.qreg rbx
  pop.qreg rax
  sub.qreg.qreg rax, rbx
  push.qreg rax
  next

defword mul, 0
  dq $ + 8
  pop.qreg rax
  pop.qreg rbx
  mul.rax.qreg rbx
  push.qreg rax
  next

defword divmod, 0
  dq $ + 8
  xor.qreg.qreg rdx, rdx         ; rdx is the high bits of the input; zero it
  pop.qreg rbx
  pop.qreg rax
  div.rdxrax.qreg rbx
  push.qreg rdx                  ; remainder
  push.qreg rax                  ; quotient
  next

;;;
;;; Comparison routines
;;; -------------------
;;;

; So. This is subtle. These comparison routines all have the same structure.
; Notice that eq and ne are commutative, whereas the others are not, so
; consider gt as the archetypical one when reasoning about correctness. To
; test if A > B, we do cmp A, B, which sets the flags the same way as
; subtracting B from A does. The mnemonic names of the condition codes are
; based on the assumption you do it in this order.
;
; We want to treat the top of the stack as the first operand, so we carefully
; pop in the appropriate order.
;
; There are both signed and unsigned variants of the condition codes. We use
; the signed ones.

defword eq, 0
  dq $ + 8
  pop.qreg rax
  pop.qreg rbx
  cmp.qreg.qreg rax, rbx
  set.breg.cc al, equal
  and.qreg.bimm rax, 0x01
  push.qreg rax
  next

defword ne, 0
  dq $ + 8
  pop.qreg rax
  pop.qreg rbx
  cmp.qreg.qreg rax, rbx
  set.breg.cc al, not_equal
  and.qreg.bimm rax, 0x01
  push.qreg rax
  next

defword gt, 0
  dq $ + 8
  pop.qreg rax
  pop.qreg rbx
  cmp.qreg.qreg rax, rbx
  set.breg.cc al, greater
  and.qreg.bimm rax, 0x01
  push.qreg rax
  next

; Is the top of the stack less than the second item in the stack?
defword lt, 0
  dq $ + 8
  pop.qreg rax
  pop.qreg rbx
  cmp.qreg.qreg rax, rbx
  set.breg.cc al, less
  and.qreg.bimm rax, 0x01
  push.qreg rax
  next

defword ge, 0
  dq $ + 8
  pop.qreg rax
  pop.qreg rbx
  cmp.qreg.qreg rax, rbx
  set.breg.cc al, greater_equal
  and.qreg.bimm rax, 0x01
  push.qreg rax
  next

defword le, 0
  dq $ + 8
  pop.qreg rax
  pop.qreg rbx
  cmp.qreg.qreg rax, rbx
  set.breg.cc al, less_equal
  and.qreg.bimm rax, 0x01
  push.qreg rax
  next

defword gt_unsigned, 0
  dq $ + 8
  pop.qreg rax
  pop.qreg rbx
  cmp.qreg.qreg rax, rbx
  set.breg.cc al, above
  and.qreg.bimm rax, 0x01
  push.qreg rax
  next

defword lt_unsigned, 0
  dq $ + 8
  pop.qreg rax
  pop.qreg rbx
  cmp.qreg.qreg rax, rbx
  set.breg.cc al, below
  and.qreg.bimm rax, 0x01
  push.qreg rax
  next

defword ge_unsigned, 0
  dq $ + 8
  pop.qreg rax
  pop.qreg rbx
  cmp.qreg.qreg rax, rbx
  set.breg.cc al, above_equal
  and.qreg.bimm rax, 0x01
  push.qreg rax
  next

defword le_unsigned, 0
  dq $ + 8
  pop.qreg rax
  pop.qreg rbx
  cmp.qreg.qreg rax, rbx
  set.breg.cc al, below_equal
  and.qreg.bimm rax, 0x01
  push.qreg rax
  next

;;;
;;; Bitwise routines
;;; ----------------
;;;

defword and, 0
  dq $ + 8
  pop.qreg rbx
  pop.qreg rax
  and.qreg.qreg rax, rbx
  push.qreg rax
  next

defword or, 0
  dq $ + 8
  pop.qreg rbx
  pop.qreg rax
  or.qreg.qreg rax, rbx
  push.qreg rax
  next

defword xor, 0
  dq $ + 8
  pop.qreg rbx
  pop.qreg rax
  xor.qreg.qreg rax, rbx
  push.qreg rax
  next

; The HP overloads the name "not", so we follow the Forth convention.
defword invert, 0
  dq $ + 8
  pop.qreg rax
  not.qreg rax
  push.qreg rax
  next

;;;
;;; Routine lit
;;; ------------
;;;

defword lit, 0
  dq $ + 8
  lodsq
  push.qreg rax
  next

defword litstring, 0
  dq $ + 8
  ; The string immediately follows the codeword in memory, so rsi is already
  ; pointing to it. That address will be our returned result, so we push it to
  ; the stack.
  push.qreg rsi

  ; Now we need to skip over the string, so that rsi will be valid for the
  ; next Forth word. To do that, we're going to do a string operation with
  ; scasb, which takes rdi as the address to look at. This means scasb is
  ; treating its operand as analogous to the destination operand of movs*.
  mov.qreg.qreg rdi, rsi
  ; We want to compare for equality with zero; scasb looks in al for the other
  ; half of the comparison, so we clear rax.
  xor.qreg.qreg rax, rax
  ; Counterintuitively, we do need to pass a count. We pass -1, which will
  ; always work.
  xor.qreg.qreg rcx, rcx
  not.qreg rcx
  ; The DF flag is zero per our Forth execution-model convention, which means
  ; scasb increments rdi after each iteration. This is what we want, so
  ; there's no need to mess with it.
  ;
  ; We also need to worry about ZF, but fortunately that "not" will make sure
  ; it's clear.
  ;
  ; So, we're ready; do it!
  repnz scasb
  ; The scasb completes, incrementing rdi, before the repnz checks its
  ; condition. So, rdi is now pointing immediately after the terminating null
  ; byte. Of course, we want this in rsi for Forth's purposes.
  mov.qreg.qreg rsi, rdi

  ; Finally, we need to align rsi to the next word boundary.
  add.qreg.bimm rsi, 7
  and.qreg.bimm rsi, NOT 7

  next

;;;
;;; Memory access routines
;;; ----------------------
;;;
;;;   We go with Forth names for this stuff. The HP's names for memory and
;;; storage operations heavily leverage the fact they have an object system
;;; with type tags and so on; we want to stay close to the bytes.
;;;

; Address on the top of the stack, value in the second position
defword store, 0
  dq $ + 8
  pop.qreg rbx
  pop.qreg rax
  mov.indirect.qreg.qreg rbx, rax
  next

defword fetch, 0
  dq $ + 8
  pop.qreg rax
  mov.qreg.indirect.qreg rax, rax
  push.qreg rax
  next

; Address on top, value second
; I might have done it the other way, but this is what Jonesforth does and it
; seems reasonable enough.
defword addstore, 0
  dq $ + 8
  pop.qreg rbx
  pop.qreg rax
  add.indirect.qreg.qreg rbx, rax
  next

defword substore, 0
  dq $ + 8
  pop.qreg rbx
  pop.qreg rax
  sub.indirect.qreg.qreg rbx, rax
  next

defword store8, 0
  dq $ + 8
  pop.qreg rbx
  pop.qreg rax
  mov.indirect.qreg.breg rbx, al
  next

defword fetch8, 0
  dq $ + 8
  pop.qreg rbx
  xor.qreg.qreg rax, rax
  mov.breg.indirect.qreg al, rbx
  push.qreg rax
  next

defword store16, 0
  dq $ + 8
  pop.qreg rbx
  pop.qreg rax
  mov.indirect.qreg.wreg rbx, ax
  next

defword fetch16, 0
  dq $ + 8
  pop.qreg rbx
  xor.qreg.qreg rax, rax
  mov.wreg.indirect.qreg ax, rbx
  push.qreg rax
  next

defword store32, 0
  dq $ + 8
  pop.qreg rbx
  pop.qreg rax
  mov.indirect.qreg.dreg rbx, eax
  next

defword fetch32, 0
  dq $ + 8
  pop.qreg rbx
  xor.qreg.qreg rax, rax
  mov.dreg.indirect.qreg eax, rbx
  push.qreg rax
  next

; Before we get too deep into it, we also define a few reflection routines
; that retrieve, or set, the address of either of the two stacks.
;
; The result, or the new value, is on the top of the data stack. That's the
; same as how it works for any other word, but given the metacircular nature
; of these ones it's easy to get confused about that...
defword store_control_stack, 0
  dq $ + 8
  pop.qreg rbp
  next
defword fetch_control_stack, 0
  dq $ + 8
  push.qreg rbp
  next
defword store_value_stack, 0
  dq $ + 8
  ; Per Intel's description of POP this reads from the old location, and there
  ; is no increment applied to the resulting value. See, the description says
  ; it increments the register then overwrites it, in that order.
  pop.qreg rsp
  next
defword fetch_value_stack, 0
  dq $ + 8
  ; Per Intel's description of PUSH this pushes the old value.
  push.qreg rsp
  next

;   Copy one non-overlapping block of memory over another. For the overlapping
; logic, see roll and unroll. This always copies at byte granularity, for ease
; of implementation.
;
;   Jonesforth calls this CMOVE and has a CCOPY which is a single byte. POSIX,
; however, has memcpy() for non-overlapping blocks and memmove() for
; overlapping blocks. We follow the latter convention, because it feels like
; the more important distinction.
;
;   Also, the "c" was meant to indicate that it works at one-byte granularity,
; but that isn't, uh... actually an important property here, and as a blanket
; call we're not using letters to denote data sizes. So we call it "memcopy".
; Apologies to the C programming tradition but vowels are good, actually.
;
;   Jonesforth also offers C@C! as another name for its CCOPY, but neither
; "@!" nor "mem@mem!" seems particulaly nice.
;
; Stack in:
;   destination
;   source
;   length
; (top)
defword memcopy, 0
  dq $ + 8
  ; We need to save and restore rsi; the other registers we can trample.
  mov.qreg.qreg rdx, rsi
  pop.qreg rcx
  pop.qreg rdi
  pop.qreg rsi
  ; We start from the low end, since that's easier arithmetic. So, we get to
  ; leave the DF flag alone.
  rep movsb
  mov.qreg.qreg rsi, rdx
  next

; (written much later)
;
; In:
;   destination
;   source
;   length
defword memmove, 0
  dq $ + 8
  ; We need to save and restore rsi; the other registers we can trample.
  mov.qreg.qreg rdx, rsi
  pop.qreg rcx
  pop.qreg rdi
  pop.qreg rsi

  ; We need to check source < destination to decide which end to start from.
  mov.qreg.qreg rax, rsi
  cmp.qreg.qreg rax, rdi
  ; Relative offsets are from the start of the instruction after the jmp.
  jmp.cc.rel.bimm below, 4

  ; If source is greater, we are sliding downwards so we start from the low
  ; end. So, we get to leave the DF flag alone.
  rep movsb                         ; 2 bytes
  jmp.rel.bimm 16                   ; 2 bytes

  ; If destination is greater, we are sliding upwards so we start from the
  ; high end. So, we have to save and restore DF. Also, we have to adjust the
  ; pointers.
  add.qreg.qreg rsi, rcx            ; 3 bytes
  dec.qreg rsi                      ; 3 bytes
  add.qreg.qreg rdi, rcx            ; 3 bytes
  dec.qreg rdi                      ; 3 bytes
  std                               ; 1 byte
  rep movsb                         ; 2 bytes
  cld                               ; 1 byte

  mov.qreg.qreg rsi, rdx
  next


; Stack in:
;   string address
; Stack out:
;   string length not including null byte
defword stringlen, 0
  dq $ + 8
  pop.qreg rdi
  mov.qreg.qreg rbx, rdi
  xor.qreg.qreg rax, rax
  xor.qreg.qreg rcx, rcx
  not.qreg rcx
  repnz scasb
  sub.qreg.qreg rdi, rbx
  sub.qreg.bimm rdi, 1
  push.qreg rdi
  next

; Stack in:
;   address of final non-null byte at the end of a string
; Stack out:
;   backwards string length not including null byte at start
defword reverse_stringlen, 0
  dq $ + 8
  pop.qreg rdi

  mov.qreg.qreg rbx, rdi
  xor.qreg.qreg rax, rax
  xor.qreg.qreg rcx, rcx
  not.qreg rcx

  std
  repnz scasb
  cld

  sub.qreg.qreg rbx, rdi
  sub.qreg.bimm rbx, 1

  push.qreg rbx
  next

;    If you have a variable-length string followed by alignment padding, and
; you want to traverse it in reverse, you also need to skip the alignment
; padding...
;
; Stack in:
;   address of final null byte in alignment padding of total length up to 8
; Stack out:
;   number of null bytes (from end) until nearest non-null byte, or 8
defword reverse_padding_len, 0
  dq $ + 8
  pop.qreg rdi

  mov.qreg.qreg rbx, rdi
  xor.qreg.qreg rax, rax
  mov.qreg.dimm rcx, 9

  std
  repz scasb
  cld

  sub.qreg.qreg rbx, rdi
  sub.qreg.bimm rbx, 1

  push.qreg rbx
  next

;   We make this work using exactly two jump instructions, which is likely the
; minimum possible. To avoid relying on labels, we hand-compute the byte
; offsets, so every instruction within their ranges is annotated with its
; length in bytes. Good fun.
;
; Stack in:
;   left string address
;   right string address
; Stack out:
;   comparison value (0 for equal, 1 for left > right, -1 for left < right)
defword stringcmp, 0
  dq $ + 8
  ;   Save the rsi register.
  ;
  ;   Happily, we don't need a lot of registers for this code, so we can
  ; dedicate rdx to this and not have to deal with stack juggling..
  mov.qreg.qreg rdx, rsi

  ;   Get the parameters off the stack.
  ;
  ;   For reasons that will be explained below, the left and right strings
  ; from our caller's perspective are swapped from the perspective of the
  ; comparisons we'll be doing. The rdi register points to the right-hand
  ; operand of the "cmps" instruction, while rsi points to the left-hand
  ; operand; [Intel] volume 2A, chapter 3, section 4-3.3, "CMPS"; note that
  ; the PDF mis-numbers the sections in the second half of chapter 3.
  pop.qreg rsi
  pop.qreg rdi

  ;   For the comparison-unequal loop-exit code, we'll need rbx set to zero.
  ; The easy way to do that is by xor'ing it with itself, but doing it at the
  ; end of the loop would overwrite the comparison flags, which we need. So we
  ; do it in advance, and carefully don't touch it anywhere else.
  ;
  ;   See the list of which instructions affect the flags in [Intel] volume 1,
  ; appendix A, section A-1, table A-2.
  xor.qreg.qreg rbx, rbx

  ;   We also clear rax; we'll be using al later, and it will be convenient to
  ; know the upper bits are always zero.
  xor.qreg.qreg rax, rax

  ; Now we've initialized everything we need; everything after this point is
  ; part of the loop.

  ;   At the start of each iteration, save a copy of the byte we're at now. We
  ; need to do this before cmps because the pointers will increment; we'll
  ; eventually use it to test whether we've reached the end delimiter.
  ;
  ;   We do a 64-bit load into rcx, which is otherwise unused, then copy the
  ; low byte from cl to al. This avoids having to think about addressing modes
  ; that combine 64-bit addresses with smaller data sizes; those are very
  ; subtle.
  mov.qreg.indirect.qreg rcx, rsi   ; 3 bytes
  mov.breg.breg al, cl              ; 2 bytes

  ;   Now do the cmps, which is the heart of the loop.
  ;
  ;   Note that, in addition to relying on this comparison to test content
  ; bytes against each other, in the event that one string is a prefix of the
  ; other, this will also test content bytes against null delimiters. The
  ; longer string will compare as "greater", because zero is less than all
  ; other possible byte values. This is what we want.
  ;
  ;   Since strings of different lengths are necessarily unequal, letting this
  ; test do the work of detecting that also means we don't have to deal with
  ; scenarios where we're past the end of one string but not the other.
  cmps8                             ; 1 byte

  ;   The flags are now set based on the simulated subtraction of the next
  ; bytes from the two strings. If they were unequal, the loop will end. If
  ; they were equal, we have another test to do before we're finished with
  ; this iteration. So we put the loop-end code next, and conditionally jump
  ; forward past it.
  ;
  ;   Recall that relative offsets are from the start of the instruction after
  ; the jmp.
  jmp.cc.rel.bimm equal, 15         ; 2 bytes

  ;   If we got here, the strings are unequal, so we need to turn the flags
  ; into a comparison value. We cleared rbx earlier; now we set its low bit to
  ; the "above" flag, then use sbb to subtract the "carry" flag (which it
  ; thinks of as the "borrow" flag).
  ;
  ;   Recall that "carry" is based on whether the comparison would cause a
  ; change to the next bit "outside" the bits being compared; [Intel]
  ; volume 1, chapter 3, section 3-4.3.1. Since the comparison is a
  ; subtraction, "carry" is true when the right-hand byte is strictly greater
  ; than the left-hand byte.
  ;
  ;   Also recall that "zero", also called "equal", is based on whether the
  ; comparison's result produces output that's all zero bits. Since the
  ; comparison is a subtraction, "zero" is true when the two bytes are
  ; identical.
  ;
  ;   The "above" condition is not a flag, but a composite of flags. It's true
  ; when both "carry" and "zero" are false; [Intel] volume 1, chapter 7,
  ; section 7-3.1.1, table 7-2. For us, this is equivalent to saying the
  ; left-hand byte is strictly greater than the right-hand.
  ;
  ;   So, after the "set" instruction, rbx is 1 if left > right, and 0 if
  ; left = right or left < right. The "sbb" instruction with an immediate
  ; value of zero subtracts the value of the carry flag from rbx; the flag is
  ; true in the case where left < right, and false otherwise. So, after the
  ; "sbb", rbx is 1 if left > right, 0 if left = right, and -1 if
  ; left < right.
  ;
  ;   This sounds like the opposite of what we want, but recall that we
  ; exchanged the operands, so these are the values our caller is expecting.
  ; While it might be tempting to look for a way to not need that
  ; transposition, and indeed there exists an approach using "not above", it's
  ; fiddly in a way that's even harder to explain.
  ;
  ;   Note that it's important that the "jmp" and "set" instructions don't
  ; change the flags (see table A-2 again), so the "cmps" is the most recent
  ; thing that did. The "sbb" instruction does change them, but we don't need
  ; them again after that point so it doesn't matter.
  ;
  ;   Finally, we need to restore rsi before we return to Forth.
  ;
  ;   The two-instruction set-sbb sequence is one of those classic assembly
  ; programming tricks that often goes unexplained, because most of its appeal
  ; is its brevity and a proper explanation is quite lengthy. Please enjoy
  ; this gift of knowledge, and thanks to our friends who showed it to us.
  set.breg.cc bl, above             ; 3 bytes
  sbb.qreg.bimm rbx, 0              ; 4 bytes
  push.qreg rbx                     ; 1 byte
  mov.qreg.qreg rsi, rdx            ; 3 bytes
  next                              ; 4 bytes

  ;   If we got here, the current bytes are equal to each other. We still need
  ; to test if they're null terminators; if so, we exit, and if not, we loop.
  ;
  ;   At the beginning of this iteration, we saved a copy of the byte being
  ; inspected in rax. The "test" instruction simulates a xor and sets the
  ; flags accordingly. We check the "not equal" condition, also known as "not
  ; zero", and jump backwards to the start of the loop when that's the case.
  ;
  ;   Recall that relative offsets are from the start of the instruction after
  ; the jmp.
  test.qreg.qreg rax, rax           ; 3 bytes
  jmp.cc.rel.bimm not_equal, -28    ; 2 bytes

  ;   If we got here, we got all the way to the end of both strings and found
  ; a null byte, so we return 0 to indicate they're equal.
  ;
  ;   We can use push.dimm even though our stack holds 64-bit values, because
  ; it gets sign-extended.
  ;
  ;   We need to restore rsi before we return to Forth.
  push.dimm 0
  mov.qreg.qreg rsi, rdx
  next


;;;;;;;;;;;;;;;;;
;;; Branching ;;;
;;;;;;;;;;;;;;;;;

;   This takes a number of bytes, not machine words. That allows it to be used
; for putting weird things embedded in the code.
;
;   The offset is relative to the start of the word the number of bytes is in,
; so, make sure to have it skip itself.
defword branch, 0
  dq $ + 8
  add.qreg.indirect.qreg rsi, rsi
  next

; This should probably be 0branch, but right now the auto-label code is picky.
defword zbranch, 0
  dq $ + 8
  pop.qreg rax
  test.qreg.qreg rax, rax
  ; Please notice the 8-bit branch to the nearby word.
  jmp.cc.rel.bimm zero, branch + 8 - zbranch_after_jmp
zbranch_after_jmp:
  lodsq                          ; just a convenient way to skip rsi forward
  next

; This is like next, but instead of using rsi as the "instruction pointer", it
; takes a codeword address from the value stack.
;
; In the event that the codeword is docol, docol will handle any manipulation
; of the control stack that needs to happen. Yes, it really is that simple.
defword execute, 0
  dq $ + 8
  pop.qreg rax
  jmp.abs.indirect.qreg rax

;;;;;;;;;;;;;;;;;;;;
;;; System calls ;;;
;;;;;;;;;;;;;;;;;;;;
;;;
;;;   The kernel preserves every register except rax, rcx, and r11. The system
;;; call number goes in rax, as does the return value. Parameters go in rdi,
;;; rsi, rdx, r10, r8, and r9, in that order. [SysV] A.2.1.
;;;
;;;   Notice that rsi is our control stack, so we have to save it (for
;;; syscalls with at least two parameters). We can use the value stack to do
;;; that, since rsp is preserved. We don't save other registers because our
;;; caller should do that, if it cares.
;;;

;;;
;;;   This does the Linux exit() system call, passing it an exit code taken
;;; from the stack.
;;;
defword sys_exit, 0
  dq $ + 8

  mov.qreg.qimm rax, 60          ; syscall number
  pop.qreg rdi                   ; exit code
  syscall

  ; In the event we're still here, let's minimize confusion.
  hlt


;;;
;;;   This does the Linux write() system call, passing it an address from the
;;; top of the stack and a length from the second position on the stack. It
;;; writes to file descriptor 1, which is stdout.
;;;
;;;   For our length parameter, we can pop directly from the stack into rdx,
;;; which directly becomes the syscall parameter. For our address parameter,
;;; the syscall wants it in rsi, which we also care about, so we have to do a
;;; little juggling.
;;;
defword sys_write, 0
  dq $ + 8
  pop.qreg rcx                   ; address from stack
  pop.qreg rdx                   ; length from stack, passed directly
  push.qreg rsi                  ; save rsi
  mov.qreg.qimm rax, 1           ; syscall number
  mov.qreg.qimm rdi, 1           ; file descriptor
  mov.qreg.qreg rsi, rcx         ; pass address
  syscall
  pop.qreg rsi                   ; restore rsi
  next


; In:
;   length
;   address
defword sys_read, 0
  dq $ + 8
  pop.qreg rcx                   ; address from stack
  pop.qreg rdx                   ; length from stack, passed directly
  push.qreg rsi                  ; save rsi
  mov.qreg.qimm rax, 0           ; syscall number
  mov.qreg.qimm rdi, 0           ; file descriptor
  mov.qreg.qreg rsi, rcx         ; pass address
  syscall
  pop.qreg rsi                   ; restore rsi
  push.qreg rax                  ; return length
  next


;;;;;;;;;;;;;;;;;;;;;;;;
;;; I/O conveniences ;;;
;;;;;;;;;;;;;;;;;;;;;;;;


defword emitstring, 0
  dq docol, dup, stringlen, swap, sys_write, exit


;   The main word for this next part is "dot"; the stuff preceding it is
; internals used to implement it.
;
;   Although this could notionally be emitinteger for symmetry, it's
; well-known under the name dot and as a single period character, and that
; name participates in conventions for names of other things. So, we go with
; it.
;
;   Unlike Jonesforth, we do not have a global "base" variable, and this word
; does not change its behavior depending on that value. It's always base ten.
;
;   This is an extremely inefficient implementation, but on the plus side,
; doing that avoids having to think about any sort of memory management or
; recursion, and lets us stick entirely with the trivial control-flow
; constructs we already have.

; In:
;   base
;   exponent
; Out:
;   base to the power of exponent
defword pow, 0
  dq docol
  dq lit, 1, swap
  ; If the count of remaining powers is NOT equal to zero, the comparison will
  ; return 0, which will cause the zbranch to skip the rest of the line.
  dq dup, lit, 0, eq, zbranch, 5*8, drop, swap, drop, exit
  ; (base, result so far, count of remaining powers)
  dq lit, 1, sub, unroll3
  ; (updated count of remaining powers, base, result so far)
  dq swap, dup, lit, 4, unroll
  ; (base, updated count of remaining powers, result so far, base)
  dq mul, swap
  ; (base, updated result so far, updated count of remaining powers)
  dq branch, -22*8

; In:
;   base
;   value
; Out:
;   floor of logarithm of value, in base base
defword logfloor, 0
  dq docol

  ; Start with a product equal to the base, and a count of 0.
  dq swap, dup, unroll3, lit, 0

  ; This is the start of the loop body.
  ; (base, value, product so far, count of powers included so far)
  dq unroll3, dup2
  ; (base, count so far, value, product so far)
  dq gt_unsigned, zbranch, 6*8

  ; If we get here, we're done.
  ; (base, count so far, value, product so far)
  dq drop, drop, swap, drop, exit

  ; If we're here, we need to do another loop.
  ; (base, count so far, value, product so far)
  dq lit, 4, roll, dup, lit, 5, unroll, mul, roll3, lit, 1, add
  ; (base, value, updated product so far, updated count so far)

  ; If the product is less than the base, we overflowed. In that case, the
  ; product-so-far is the maximum, so just return it.
  dq lit, 4, roll, dup, lit, 5, unroll, roll3, dup, lit, 4, unroll
  dq lt, zbranch, 8*8
  dq lit, 4, unroll, drop, drop, drop, exit

  ; Nothing else weird going on, so loop.
  dq branch, -45*8

; In:
;   base
;   value
; Out:
;   celing of logarithm of value, in base base
defword logceil, 0
  dq docol

  ; Start with a product of 1 and a count of 0.
  dq lit, 1, lit, 0

  ; This is the start of the loop body.
  ; (base, value, product so far, count of powers included so far)
  dq unroll3, dup2
  ; (base, count so far, value, product so far)
  dq ge_unsigned, zbranch, 6*8

  ; If we get here, we're done.
  ; (base, count so far, value, product so far)
  dq drop, drop, swap, drop, exit

  ; If we're here, we need to do another loop.
  ; (base, count so far, value, product so far)
  dq lit, 4, roll, dup, lit, 5, unroll, mul, roll3, lit, 1, add
  ; (base, value, updated product so far, updated count so far)

  ; If the product is less than the base, we overflowed. In that case, the
  ; product-so-far is the maximum, so just return it.
  dq lit, 4, roll, dup, lit, 5, unroll, roll3, dup, lit, 4, unroll
  dq lt, zbranch, 8*8
  dq lit, 4, unroll, drop, drop, drop, exit

  ; Nothing else weird going on, so loop.
  dq branch, -45*8

; In:
;   integer to print
;   base to print in
;   width to zero-pad to
defword dotbase_unsigned, 0
  dq docol

  ; (input, base, width)
  ; Compute how many digits we need to display. Because we use logfloor, the
  ; logic of always printing at least one digit is already handled for us.
  dq unroll3, swap, dup2, logfloor, lit, 1, add
  ; (width, base, input, number of digits if no padding)
  dq lit, 4, roll, dup2, gt, zbranch, 5*8
  ; (base, input, number of digits if no padding, width)
  ; If we're here, we should use the padded width
  dq swap, drop, branch, 2*8
  ; (base, input, number of digits if no padding, width)
  ; If we're here, we should use the unpadded width
  dq drop

  ; (base, input, number of digits remaining)
  ; This is the start of the loop.
  dq dup2, lit, 1, sub
  ; (base, input, number of digits remaining, input, intermediate value)
  dq lit, 5, roll, dup, lit, 6, unroll, swap
  ; (base, input, number of digits remaining, input, base, intermediate value)
  dq pow, divmod, swap, drop
  ; (base, input, number of digits remaining,
  ;  input divided by base^x appropriately)
  dq lit, 4, roll, dup, lit, 5, unroll
  ; (base, input, number of digits remaining,
  ;  input divided by base^x appropriately, base)
  dq divmod, drop

  ; (base, input, number of digits remaining, current digit)
  ; We construct a one-character string on the stack, then use a pointer to
  ; it. It will always contain its own null-termination without us having to
  ; do anything special to that end.
  dq dup, lit, 10, gt, zbranch, 5*8
  dq lit, "0", branch, 6*8
  dq lit, 10, sub, lit, "a"
  dq add
  dq fetch_value_stack, emitstring

  ; We deallocate the string by dropping it.
  ;
  ; Saying it like that makes it sound obvious; contemplate it until it feels
  ; surprising.
  dq drop

  dq lit, 1, sub
  dq dup, zbranch, 3*8, branch, -51*8
  dq drop, drop, drop, exit

; In:
;   integer to print
;   base to print in
defword dotbase, 0
  dq docol

  dq swap
  ; (base, input)

  ; Deal with negative numbers.
  dq dup, lit, 0, gt, zbranch, 7*8
  dq lit, -1, mul
  dq litstring, "-", emitstring

  dq swap, lit, 0, dotbase_unsigned, exit

; In:
;   integer to print
defword dot, 0
  dq docol, lit, 10, dotbase, exit
defword dothex, 0
  dq docol, lit, 16, lit, 0, dotbase_unsigned, exit
defword dothex8, 0
  dq docol, lit, 16, lit, 2, dotbase_unsigned, exit
defword dothex16, 0
  dq docol, lit, 16, lit, 4, dotbase_unsigned, exit
defword dothex32, 0
  dq docol, lit, 16, lit, 8, dotbase_unsigned, exit
defword dothex64, 0
  dq docol, lit, 16, lit, 16, dotbase_unsigned, exit


;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; Development utilities ;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

;   This peforms the "hlt" instruction (Intel's mnemomic, short for "halt"),
; which will cause the program to exit with a segmentation fault. If you're
; running under a debugger, this is a convenient way to get execution to stop
; at a certain point.
;
;   It's called "crash" rather than "hlt" to distinguish it from the word
; which outputs the instruction as machine code.
defword crash, 0
  dq $ + 8
  hlt


;;;;;;;;;;;;;;;;;;;;;;
;;; Output helpers ;;;
;;;;;;;;;;;;;;;;;;;;;;
;;;
;;;   These routines are for building up data structures in-memory. Sometimes
;;; they're used for structures that are meant to stay in memory; other times
;;; it's a buffer that will become output.
;;;
;;;   The general pattern is that each routine takes an output address and
;;; some specific datum, and returns the output address adjusted to point
;;; after the new datum. That makes them easy to chain together.

; In: base address, value
; Out: new base address
defword pack64, 0
  dq docol
  dq swap, dup, unroll3, store, lit, 8, add
  dq exit
defword pack32, 0
  dq docol
  dq swap, dup, unroll3, store32, lit, 4, add
  dq exit
defword pack16, 0
  dq docol
  dq swap, dup, unroll3, store16, lit, 2, add
  dq exit
defword pack8, 0
  dq docol
  dq swap, dup, unroll3, store8, lit, 1, add
  dq exit

;   This works on C-style strings, which are characters followed by a null
; terminator. The packed data includes the null terminator.
;
; Stack in:
;   base address, string pointer
; Stack out:
;   new base address
defword packstring, 0
  dq docol
  dq dup, stringlen, lit, 1, add, dup
  ; base/destination, source, length, length
  dq lit, 4, roll, dup, lit, 4, unroll
  ; source, destination, length, length, base/destination
  dq add, lit, 4, unroll
  ; new base, source, destination, length
  dq memcopy
  dq exit

;   This also works on C-style strings, relying on the null terminator, but
; the packed data does not include it.
;
; Stack in:
;   base address, string pointer
; Stack out:
;   new base address
defword pack_raw_string, 0
  dq docol
  dq dup, stringlen, dup
  ; base/destination, source, length, length
  dq lit, 4, roll, dup, lit, 4, unroll
  ; source, destination, length, length, base/destination
  dq add, lit, 4, unroll
  ; new base, source, destination, length
  dq memcopy
  dq exit

; Stack in:
;   base address
;   byte size
; Stack out:
;   new base address
defword packalign, 0
  dq docol
  dq dup2, divmod, drop, zbranch, 8*8
  dq swap, lit, 0, pack8, swap
  dq branch, -11*8
  dq drop, exit

; In the interests of reducing our executable's size, since a lot of it goes
; to pack* invocations, we define words that combine lit with pack*. This
; shaves roughly 700 bytes as of when it was added.
defword litpack64, 0
  dq $ + 8
  lodsq
  push.qreg rax
  beforenext pack64
defword litpack32, 0
  dq $ + 8
  lodsq
  push.qreg rax
  beforenext pack32
defword litpack16, 0
  dq $ + 8
  lodsq
  push.qreg rax
  beforenext pack16
defword litpack8, 0
  dq $ + 8
  lodsq
  push.qreg rax
  beforenext pack8


;;;;;;;;;;;;;;;;;;;;;
;;; Input helpers ;;;
;;;;;;;;;;;;;;;;;;;;;
;;;
;;;   These routines are for examining data structures in-memory.
;;;
;;;   Similarly to the output routines, each routine takes an input address,
;;; which it updates to point after the data item being read. Since this is
;;; input, the routines return data items rather than accepting them.

; In:
;   proposed size
;   alignment
; Out:
;   adjusted size
defword align_size, 0
  dq docol
  dq dup, unroll3, dup, unroll3
  ; (alignment, proposed size, alignment)
  dq lit, 1, sub, add, swap, divmod, swap, drop, mul
  dq exit

; In: base address
; Out: new base address, value
defword unpack64, 0
  dq docol
  dq dup, fetch, swap, lit, 8, add, swap
  dq exit
defword unpack32, 0
  dq docol
  dq dup, fetch32, swap, lit, 4, add, swap
  dq exit
defword unpack16, 0
  dq docol
  dq dup, fetch16, swap, lit, 2, add, swap
  dq exit
defword unpack8, 0
  dq docol
  dq dup, fetch8, swap, lit, 1, add, swap
  dq exit

;   You might think this would be identical to packalign, but packalign has
; side effects.
;
; Stack in:
;   base address
;   byte size
; Stack out:
;   new base address
defword unpackalign, 0
  dq docol, align_size, exit


;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; Assembly language, but in Forth ;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;
;;;   It's all backwards and stuff.
;;;
;;;   Okay, but seriously, the convention is: target on the top of the stack,
;;; source behind it. This is similar to how the Forth "store" and "fetch"
;;; words work.
;;;
;;;   These routines use the output helpers, defined above.  They're called in
;;; the same way: an output address, followed by data items specific to what's
;;; being output.

; We define a bunch of keywords, which evaluate to their own codeword
; addresses.
defword rax, 0
  dq docol, lit, rax, exit
defword rcx, 0
  dq docol, lit, rcx, exit
defword rdx, 0
  dq docol, lit, rdx, exit
defword rbx, 0
  dq docol, lit, rbx, exit
defword rsp, 0
  dq docol, lit, rsp, exit
defword rbp, 0
  dq docol, lit, rbp, exit
defword rsi, 0
  dq docol, lit, rsi, exit
defword rdi, 0
  dq docol, lit, rdi, exit
defword r8, 0
  dq docol, lit, r8, exit
defword r9, 0
  dq docol, lit, r9, exit
defword r10, 0
  dq docol, lit, r10, exit
defword r11, 0
  dq docol, lit, r11, exit
defword r12, 0
  dq docol, lit, r12, exit
defword r13, 0
  dq docol, lit, r13, exit
defword r14, 0
  dq docol, lit, r14, exit
defword r15, 0
  dq docol, lit, r15, exit
defword eax, 0
  dq docol, lit, eax, exit
defword ecx, 0
  dq docol, lit, ecx, exit
defword edx, 0
  dq docol, lit, edx, exit
defword ebx, 0
  dq docol, lit, ebx, exit
defword esp, 0
  dq docol, lit, esp, exit
defword ebp, 0
  dq docol, lit, ebp, exit
defword esi, 0
  dq docol, lit, esi, exit
defword edi, 0
  dq docol, lit, edi, exit
defword ax, 0
  dq docol, lit, ax, exit
defword cx, 0
  dq docol, lit, cx, exit
defword dx, 0
  dq docol, lit, dx, exit
defword bx, 0
  dq docol, lit, bx, exit
defword sp, 0
  dq docol, lit, sp, exit
defword bp, 0
  dq docol, lit, bp, exit
defword si, 0
  dq docol, lit, si, exit
defword di, 0
  dq docol, lit, di, exit
defword al, 0
  dq docol, lit, al, exit
defword cl, 0
  dq docol, lit, cl, exit
defword dl, 0
  dq docol, lit, dl, exit
defword bl, 0
  dq docol, lit, bl, exit
defword ah, 0
  dq docol, lit, ah, exit
defword ch, 0
  dq docol, lit, ch, exit
defword dh, 0
  dq docol, lit, dh, exit
defword bh, 0
  dq docol, lit, bh, exit
defword cc_overflow, 0
  dq docol, lit, cc_overflow, exit
defword cc_no_overflow, 0
  dq docol, lit, cc_no_overflow, exit
defword cc_below, 0
  dq docol, lit, cc_below, exit
defword cc_above_equal, 0
  dq docol, lit, cc_above_equal, exit
defword cc_equal, 0
  dq docol, lit, cc_equal, exit
defword cc_not_equal, 0
  dq docol, lit, cc_not_equal, exit
defword cc_below_equal, 0
  dq docol, lit, cc_below_equal, exit
defword cc_above, 0
  dq docol, lit, cc_above, exit
defword cc_sign, 0
  dq docol, lit, cc_sign, exit
defword cc_not_sign, 0
  dq docol, lit, cc_not_sign, exit
defword cc_even, 0
  dq docol, lit, cc_even, exit
defword cc_odd, 0
  dq docol, lit, cc_odd, exit
defword cc_less, 0
  dq docol, lit, cc_less, exit
defword cc_greater_equal, 0
  dq docol, lit, cc_greater_equal, exit
defword cc_less_equal, 0
  dq docol, lit, cc_less_equal, exit
defword cc_greater, 0
  dq docol, lit, cc_greater, exit

; Stack in:
;   register name
; Stack out:
;   3-bit encoded value for register
defword reg64, 0
  dq docol
  dq dup, rax, eq, zbranch, 5*8, drop, lit, 0, exit
  dq dup, rcx, eq, zbranch, 5*8, drop, lit, 1, exit
  dq dup, rdx, eq, zbranch, 5*8, drop, lit, 2, exit
  dq dup, rbx, eq, zbranch, 5*8, drop, lit, 3, exit
  dq dup, rsp, eq, zbranch, 5*8, drop, lit, 4, exit
  dq dup, rbp, eq, zbranch, 5*8, drop, lit, 5, exit
  dq dup, rsi, eq, zbranch, 5*8, drop, lit, 6, exit
  dq dup, rdi, eq, zbranch, 5*8, drop, lit, 7, exit
  dq litstring, "Parameter to reg64 is not a reg64.", emitstring
  dq lit, 1, sys_exit

; Stack in:
;   register name
; Stack out:
;   3-bit encoded value for register
defword extrareg64, 0
  dq docol
  dq dup, r8, eq, zbranch, 5*8, drop, lit, 0, exit
  dq dup, r9, eq, zbranch, 5*8, drop, lit, 1, exit
  dq dup, r10, eq, zbranch, 5*8, drop, lit, 2, exit
  dq dup, r11, eq, zbranch, 5*8, drop, lit, 3, exit
  dq dup, r12, eq, zbranch, 5*8, drop, lit, 4, exit
  dq dup, r13, eq, zbranch, 5*8, drop, lit, 5, exit
  dq dup, r14, eq, zbranch, 5*8, drop, lit, 6, exit
  dq dup, r15, eq, zbranch, 5*8, drop, lit, 7, exit
  dq litstring, "Parameter to extrareg64 is not an extrareg64.", emitstring
  dq lit, 1, sys_exit

; Stack in:
;   register name
; Stack out:
;   3-bit encoded value for register
defword reg32, 0
  dq docol
  dq dup, eax, eq, zbranch, 5*8, drop, lit, 0, exit
  dq dup, ecx, eq, zbranch, 5*8, drop, lit, 1, exit
  dq dup, edx, eq, zbranch, 5*8, drop, lit, 2, exit
  dq dup, ebx, eq, zbranch, 5*8, drop, lit, 3, exit
  dq dup, esp, eq, zbranch, 5*8, drop, lit, 4, exit
  dq dup, ebp, eq, zbranch, 5*8, drop, lit, 5, exit
  dq dup, esi, eq, zbranch, 5*8, drop, lit, 6, exit
  dq dup, edi, eq, zbranch, 5*8, drop, lit, 7, exit
  dq litstring, "Parameter to reg32 is not a reg32.", emitstring
  dq lit, 1, sys_exit

; Stack in:
;   register name
; Stack out:
;   3-bit encoded value for register
defword reg16, 0
  dq docol
  dq dup, ax, eq, zbranch, 5*8, drop, lit, 0, exit
  dq dup, cx, eq, zbranch, 5*8, drop, lit, 1, exit
  dq dup, dx, eq, zbranch, 5*8, drop, lit, 2, exit
  dq dup, bx, eq, zbranch, 5*8, drop, lit, 3, exit
  dq dup, sp, eq, zbranch, 5*8, drop, lit, 4, exit
  dq dup, bp, eq, zbranch, 5*8, drop, lit, 5, exit
  dq dup, si, eq, zbranch, 5*8, drop, lit, 6, exit
  dq dup, di, eq, zbranch, 5*8, drop, lit, 7, exit
  dq litstring, "Parameter to reg16 is not a reg16.", emitstring
  dq lit, 1, sys_exit

; Stack in:
;   register name
; Stack out:
;   3-bit encoded value for register
defword reg8, 0
  dq docol
  dq dup, al, eq, zbranch, 5*8, drop, lit, 0, exit
  dq dup, cl, eq, zbranch, 5*8, drop, lit, 1, exit
  dq dup, dl, eq, zbranch, 5*8, drop, lit, 2, exit
  dq dup, bl, eq, zbranch, 5*8, drop, lit, 3, exit
  dq dup, ah, eq, zbranch, 5*8, drop, lit, 4, exit
  dq dup, ch, eq, zbranch, 5*8, drop, lit, 5, exit
  dq dup, dh, eq, zbranch, 5*8, drop, lit, 6, exit
  dq dup, bh, eq, zbranch, 5*8, drop, lit, 7, exit
  dq litstring, "Parameter to reg8 is not a reg8.", 0, emitstring
  dq lit, 1, sys_exit

; Stack in:
;   scale factor, as a count of bytes
; Stack out:
;   2-bit encoded value for scale field in SIB byte
defword scalefield, 0
  dq docol
  dq dup, lit, 1, eq, zbranch, 5*8, drop, lit, 0, exit
  dq dup, lit, 2, eq, zbranch, 5*8, drop, lit, 1, exit
  dq dup, lit, 4, eq, zbranch, 5*8, drop, lit, 2, exit
  dq dup, lit, 8, eq, zbranch, 5*8, drop, lit, 3, exit
  dq litstring, "Parameter to scalefield is not 1, 2, 4, or 8.", emitstring
  dq lit, 1, sys_exit

; [Intel] volume 2D, appendix B, section B-1.4.7, table B-10. Also see the
; individual opcode pages.
;
; Stack in:
;   condition name
; Stack out:
;   4-bit encoded value for condition code in opcodes
defword conditioncode, 0
  dq docol
  dq dup, cc_overflow, eq, zbranch, 5*8, drop, lit, 0, exit
  dq dup, cc_no_overflow, eq, zbranch, 5*8, drop, lit, 1, exit
  dq dup, cc_below, eq, zbranch, 5*8, drop, lit, 2, exit
  dq dup, cc_above_equal, eq, zbranch, 5*8, drop, lit, 3, exit
  dq dup, cc_equal, eq, zbranch, 5*8, drop, lit, 4, exit
  dq dup, cc_not_equal, eq, zbranch, 5*8, drop, lit, 5, exit
  dq dup, cc_below_equal, eq, zbranch, 5*8, drop, lit, 6, exit
  dq dup, cc_above, eq, zbranch, 5*8, drop, lit, 7, exit
  dq dup, cc_sign, eq, zbranch, 5*8, drop, lit, 8, exit
  dq dup, cc_not_sign, eq, zbranch, 5*8, drop, lit, 9, exit
  dq dup, cc_even, eq, zbranch, 5*8, drop, lit, 10, exit
  dq dup, cc_odd, eq, zbranch, 5*8, drop, lit, 11, exit
  dq dup, cc_less, eq, zbranch, 5*8, drop, lit, 12, exit
  dq dup, cc_greater_equal, eq, zbranch, 5*8, drop, lit, 13, exit
  dq dup, cc_less_equal, eq, zbranch, 5*8, drop, lit, 14, exit
  dq dup, cc_greater, eq, zbranch, 5*8, drop, lit, 15, exit
  dq litstring, "Parameter to conditioncode is not a condition code."
  dq emitstring
  dq lit, 1, sys_exit

; Stack:
;   output point
defword rex_w, 0
  dq docol, lit, 0x48, pack8, exit
defword rex_b, 0
  dq docol, lit, 0x41, pack8, exit
defword rex_wb, 0
  dq docol, lit, 0x49, pack8, exit

; Stack:
;   output point
;   3-bit encoded value for register
;   opcode byte
defword opcodereg, 0
  dq docol, or, pack8, exit

; Stack:
;   output point
;   4-bit encoded value for condition code
;   opcode byte
defword opcodecc, 0
  dq docol, or, pack8, exit

;   The low-level word that outputs a modrm byte given fully-processed,
; numeric values for its fields. Most code will want to call one of the
; higher-level modrm_* words, instead.
;
; Stack
;   output point
;   mode ("mod") field
;   register/opcode field
;   register/memory ("RM") field
defword modrm, 0
  dq docol, swap, lit, 8, mul, or, swap, lit, 64, mul, or, pack8, exit

; Stack
;   output point
;   scale field
;   index field
;   base field
defword sib, 0
  dq docol, swap, lit, 8, mul, or, swap, lit, 64, mul, or, pack8, exit

;   The simplest of the modrm modes: Direct register addressing. There are no
; special cases to check.
;
;   It's important to notice that the R/M field may describe either a source,
; or a target, depending on what the instruction is. So, this helper doesn't
; get to know that. It also doesn't get to know whether the value in the
; reg/op field describes a register, or if instead it's an extension of the
; opcode. The caller is responsible for figuring that all out.
;
; Stack:
;   output point
;   reg/op field value (raw number)
;   reg/mem field register name
defword addressing_reg64, 0
  dq docol, reg64, lit, 3, unroll3, modrm, exit

;   This is a helper for assembly instructions that want to do a form of
; addressing that requires a value of 1 in the modrm byte's mode field, and
; do not want to do any indexing.
;
;   Its main responsibility is to deal with the scenario that requires an SIB
; byte, which happens when the R/M field has a value of 4, which would
; otherwise refer to the register rsp. In that situation, it also generates
; an SIB byte which indicates a scale of 1, no indexing, and rsp as the base
; register.
;
; Stack:
;   output point
;   reg/op field value (raw number)
;   reg/mem field register name
defword addressing_indirect_reg64, 0
  dq docol
  ; Exit with an error if the R/M register is rbp.
  dq dup, rbp, ne, zbranch, 23*8
  ; Check whether the R/M register is rsp; save the test result for later.
  dq dup, rsp, eq, lit, 4, unroll
  ; (equality result, output point, reg/op value, reg/mem name)
  dq reg64, lit, 0, unroll3, modrm
  ; (equality result, output point)
  ; If the R/M register was rsp, we need an SIB byte; otherwise, skip it.
  dq swap, zbranch, 8*8, lit, 0, lit, 4, rsp, reg64, sib
  dq exit
  dq litstring, "R/M parameter to addressing_indirect_reg64 is rbp."
  dq emitstring
  dq lit, 1, sys_exit

; Stack:
;   output point
;   reg/op field value (raw number)
;   reg/mem field register name
;   displacement value
defword addressing_disp8_reg64, 0
  dq docol
  ; This mode can do rbp fine, so no need to check for that.
  ; Check whether the R/M register is rsp; save the test result for later.
  dq swap, dup, rsp, eq, lit, 5, unroll, swap
  ; Stash the displacement value out of the way, too.
  dq lit, 4, unroll
  dq reg64, lit, 1, unroll3, modrm
  ; If the R/M register was rsp, we need an SIB byte; otherwise, skip it.
  dq roll3, zbranch, 8*8, lit, 0, lit, 4, rsp, reg64, sib
  ; The displacement byte.
  dq swap, pack8
  dq exit

; Stack:
;   output point
;   reg/op field value (raw number)
;   reg/mem field register name
;   displacement value
defword addressing_disp32_reg64, 0
  dq docol
  ; This mode can do rbp fine, so no need to check for that.
  ; Check whether the R/M register is rsp; save the test result for later.
  dq swap, dup, rsp, eq, lit, 5, unroll, swap
  ; Stash the displacement value out of the way, too.
  dq lit, 4, unroll
  dq reg64, lit, 2, unroll3, modrm
  ; If the R/M register was rsp, we need an SIB byte; otherwise, skip it.
  dq roll3, zbranch, 8*8, lit, 0, lit, 4, rsp, reg64, sib
  ; The displacement value.
  dq swap, pack32
  dq exit

; Stack:
;   output point
;   reg/op field value (raw number)
;   scale factor, as a count of bytes
;   index register name
;   base field register name
defword addressing_indexed_reg64, 0
  dq docol
  ; Exit with an error if the base register is rbp.
  dq dup, rbp, ne, zbranch, 23*8
  ; Reg/mem value 4 means to use an SIB byte (at least, with this mode).
  dq lit, 5, roll, lit, 0, lit, 6, roll, lit, 4, modrm, lit, 4, unroll
  dq reg64, unroll3, reg64, unroll3, scalefield, unroll3, sib
  dq exit
  dq litstring, "Base parameter to addressing_indexed_reg64 is rbp."
  dq emitstring
  dq lit, 1, sys_exit

; Stack:
;   output point
;   reg/op field value (raw number)
;   scale factor, as a count of bytes
;   index register name
;   base field register name
;   displacement value
defword addressing_disp8_indexed_reg64, 0
  dq docol
  ; This mode can do rbp fine, so no need to check for that.
  ; Reg/mem value 4 means to use an SIB byte (at least, with this mode).
  dq lit, 6, roll, lit, 1, lit, 7, roll, lit, 4, modrm, lit, 5, unroll
  dq lit, 5, unroll, reg64, unroll3, reg64, unroll3, scalefield, unroll3, sib
  dq swap, pack8
  dq exit

; Stack:
;   output point
;   reg/op field value (raw number)
;   reg/mem field register name
defword addressing_reg8, 0
  dq docol, reg8, lit, 3, unroll3, modrm, exit

; Stack:
;   output point
defword cld, 0
  dq docol, lit, 0xFC, pack8, exit

; Stack:
;   output point
defword std, 0
  dq docol, lit, 0xFD, pack8, exit

; Stack:
;   output point
;   immediate value
;   register name
defword mov_reg64_imm32, 0
  dq docol
  dq roll3, rex_w, lit, 0xC7, pack8, swap
  dq lit, 0, swap, addressing_reg64
  dq swap, pack32
  dq exit

; Stack:
;   output point
;   immediate value
;   register name
defword mov_reg64_imm64, 0
  dq docol
  dq roll3, rex_w, swap, reg64, lit, 0xB8, opcodereg, swap, pack64
  dq exit
defword mov_extrareg64_imm64, 0
  dq docol
  dq roll3, rex_wb, swap, extrareg64, lit, 0xB8, opcodereg, swap, pack64
  dq exit

; Stack:
;   output point
;   source register name
;   target register name
defword mov_reg64_reg64, 0
  dq docol
  dq roll3, rex_w, lit, 0x89, pack8, unroll3
  dq swap, reg64, swap, addressing_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   target register name
defword mov_indirect_reg64_reg64, 0
  dq docol
  dq roll3, rex_w, lit, 0x89, pack8, unroll3
  dq swap, reg64, swap, addressing_indirect_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   target register name
;   target displacement value
defword mov_disp8_reg64_reg64, 0
  dq docol
  dq lit, 4, roll, rex_w, lit, 0x89, pack8, lit, 4, unroll
  dq roll3, reg64, unroll3, addressing_disp8_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   target register name
defword mov_reg64_indirect_reg64, 0
  dq docol
  dq roll3, rex_w, lit, 0x8B, pack8, unroll3
  dq reg64, swap, addressing_indirect_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   source displacement value
;   target register name
defword mov_reg64_disp8_reg64, 0
  dq docol
  dq lit, 4, roll, rex_w, lit, 0x8B, pack8, lit, 4, unroll
  dq reg64, unroll3, addressing_disp8_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   source displacement value
;   target register name
defword mov_reg64_disp32_reg64, 0
  dq docol
  dq lit, 4, roll, rex_w, lit, 0x89, pack8, lit, 4, unroll
  dq roll3, reg64, swap, roll3, addressing_disp32_reg64
  dq exit

; Stack:
;   output point
;   source base register name
;   source index register name
;   source index scale factor, as a count of bytes
;   target register name
defword mov_reg64_indexed_reg64, 0
  dq docol
  dq lit, 5, roll, rex_w, lit, 0x8B, pack8, lit, 5, unroll
  dq reg64, lit, 4, unroll, unroll3, swap, addressing_indexed_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   target base register name
;   target index register name
;   target index scale factor, as a count of bytes
defword mov_indexed_reg64_reg64, 0
  dq docol
  dq lit, 5, roll, rex_w, lit, 0x89, pack8, lit, 5, unroll
  dq lit, 4, roll, reg64, lit, 4, unroll
  dq unroll3, swap, addressing_indexed_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   target register name
defword mov_indirect_reg64_reg32, 0
  dq docol
  dq roll3, lit, 0x89, pack8, unroll3
  dq swap, reg32, swap, addressing_indirect_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   target register name
;   target displacement value
defword mov_disp8_reg64_reg32, 0
  dq docol
  dq lit, 4, roll, lit, 0x89, pack8, lit, 4, unroll
  dq roll3, reg32, unroll3, addressing_disp8_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   target register name
defword mov_reg32_indirect_reg64, 0
  dq docol
  dq roll3, lit, 0x8B, pack8, unroll3
  dq reg32, swap, addressing_indirect_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   source displacement value
;   target register name
defword mov_reg32_disp8_reg64, 0
  dq docol
  dq lit, 4, roll, lit, 0x8B, pack8, lit, 4, unroll
  dq reg32, unroll3, addressing_disp8_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   target register name
defword mov_indirect_reg64_reg16, 0
  dq docol
  dq roll3, lit, 0x66, pack8, lit, 0x89, pack8, unroll3
  dq swap, reg16, swap, addressing_indirect_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   target register name
;   target displacement value
defword mov_disp8_reg64_reg16, 0
  dq docol
  dq lit, 4, roll, lit, 0x66, pack8, lit, 0x89, pack8, lit, 4, unroll
  dq roll3, reg16, unroll3, addressing_disp8_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   target register name
defword mov_reg16_indirect_reg64, 0
  dq docol
  dq roll3, lit, 0x66, pack8, lit, 0x8B, pack8, unroll3
  dq reg16, swap, addressing_indirect_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   source displacement value
;   target register name
defword mov_reg16_disp8_reg64, 0
  dq docol
  dq lit, 4, roll, lit, 0x66, pack8, lit, 0x8B, pack8, lit, 4, unroll
  dq reg16, unroll3, addressing_disp8_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   target register name
defword mov_indirect_reg64_reg8, 0
  dq docol
  dq roll3, lit, 0x88, pack8, unroll3
  dq swap, reg8, swap, addressing_indirect_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   target register name
;   target displacement value
defword mov_disp8_reg64_reg8, 0
  dq docol
  dq lit, 4, roll, lit, 0x88, pack8, lit, 4, unroll
  dq roll3, reg8, unroll3, addressing_disp8_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   target register name
defword mov_reg8_indirect_reg64, 0
  dq docol
  dq roll3, lit, 0x8A, pack8, unroll3
  dq reg8, swap, addressing_indirect_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   source displacement value
;   target register name
defword mov_reg8_disp8_reg64, 0
  dq docol
  dq lit, 4, roll, pack8, lit, 0x8A, pack8, lit, 4, unroll
  dq reg8, unroll3, addressing_disp8_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   target register name
defword mov_reg8_reg8, 0
  dq docol
  dq roll3, lit, 0x88, pack8, unroll3
  dq swap, reg8, swap, addressing_reg8
  dq exit

; Stack:
;   output point
;   source register name
;   source displacement value
;   target register name
defword lea_reg64_disp8_reg64, 0
  dq docol
  dq lit, 4, roll, rex_w, lit, 0x8D, pack8, lit, 4, unroll
  dq reg64, unroll3, addressing_disp8_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   source displacement value
;   target register name
defword lea_reg64_disp32_reg64, 0
  dq docol
  dq lit, 4, roll, rex_w, lit, 0x8D, pack8, lit, 4, unroll
  dq reg64, unroll3, addressing_disp32_reg64
  dq exit

; Stack:
;   output point
;   source base register name
;   source index register name
;   source index scale factor, as a count of bytes
;   target register name
defword lea_reg64_indexed_reg64, 0
  dq docol
  dq lit, 5, roll, rex_w, lit, 0x8D, pack8, lit, 5, unroll
  dq reg64, lit, 4, unroll, unroll3, swap, addressing_indexed_reg64
  dq exit

; Stack:
;   output point
;   source base register name
;   source index register name
;   source index scale factor, as a count of bytes
;   source displacement value
;   target register name
defword lea_reg64_disp8_indexed_reg64, 0
  dq docol
  dq lit, 6, roll, rex_w, lit, 0x8D, pack8, lit, 6, unroll
  dq reg64, lit, 5, unroll, lit, 3, roll, lit, 4, roll, lit, 3, roll
  dq addressing_disp8_indexed_reg64
  dq exit

; Stack:
;   output point
;   source register name
defword push_reg64, 0
  dq docol, reg64, lit, 0x50, opcodereg, exit

; Stack:
;   output point
;   source register name
defword push_extrareg64, 0
  dq docol, swap, rex_b, swap, extrareg64, lit, 0x50, opcodereg, exit

; Stack:
;   output point
;   target register name
defword pop_reg64, 0
  dq docol, reg64, lit, 0x58, opcodereg, exit

; Stack:
;   output point
;   target register name
defword pop_extrareg64, 0
  dq docol, swap, rex_b, swap, extrareg64, lit, 0x58, opcodereg, exit

; Stack:
;   output point
;   immediate value
defword push_imm32_extended64, 0
  dq docol, swap, lit, 0x68, pack8, swap, pack32, exit


;;;
;;; String instructions
;;;
;;;   What makes these useful is that they take their parameters from certain
;;; fixed registers, which are chosen such that the operations chain into each
;;; other well. Thus you can use them to build various block-memory and string
;;; operations, and even if you need unusual forms of loop unrolling or
;;; alignment tweaking, the code will end up uniform in structure. On modern
;;; processors, this is even the high-performance approach, due to highly
;;; optimized microcode, though these operations were inefficient when they
;;; were first invented.
;;;
;;;   We break with the Intel mnemonics, which follow the pattern
;;; movsb/movsw/movsd/movsq, because this would otherwise be the only place we
;;; use the b/w/d/q thing instead of 8/16/32/64. Tradition and
;;; pronounceability are both nice things, but approachability to newcomers is
;;; important, too.
;;;
;;;   Some of these are repeatable; whether you view the repeatable variants
;;; as different instructions is up to you. At any rate the machine code
;;; representation of the repeatable variants is the same as for the regular
;;; variants with an extra prefix, so we define them together.
;;;
;;;   This is a proper superset of the flatassembler implementations of string
;;; instructions. The wisdom of that is questionable, but at least it's noted
;;; here...

; Stack:
;   output point
defword movs8, 0
  dq docol, lit, 0xA4, pack8, exit
defword movs16, 0
  dq docol, lit, 0x66, pack8, lit, 0xA5, pack8, exit
defword movs32, 0
  dq docol, lit, 0xA5, pack8, exit
defword movs64, 0
  dq docol, rex_w, lit, 0xA5, pack8, exit
defword rep_movs8, 0
  dq docol, lit, 0xF3, pack8, lit, 0xA4, pack8, exit
defword rep_movs16, 0
  dq docol, lit, 0xF3, pack8, lit, 0x66, pack8, lit, 0xA5, pack8, exit
defword rep_movs32, 0
  dq docol, lit, 0xF3, pack8, lit, 0xA5, pack8, exit
defword rep_movs64, 0
  dq docol, lit, 0xF3, pack8, rex_w, lit, 0xA5, pack8, exit

; Stack:
;   output point
defword lods8, 0
  dq docol, lit, 0xAC, pack8, exit
defword lods16, 0
  dq docol, lit, 0x66, pack8, lit, 0xAD, pack8, exit
defword lods32, 0
  dq docol, lit, 0xAD, pack8, exit
defword lods64, 0
  dq docol, rex_w, lit, 0xAD, pack8, exit
defword rep_lods8, 0
  dq docol, lit, 0xF3, pack8, lit, 0xAC, pack8, exit
defword rep_lods16, 0
  dq docol, lit, 0xF3, pack8, lit, 0x66, pack8, lit, 0xAD, pack8, exit
defword rep_lods32, 0
  dq docol, lit, 0xF3, pack8, lit, 0xAD, pack8, exit
defword rep_lods64, 0
  dq docol, lit, 0xF3, pack8, rex_w, lit, 0xAD, pack8, exit

; Stack:
;   output pint
defword stos8, 0
  dq docol, lit, 0xAA, pack8, exit
defword stos16, 0
  dq docol, lit, 0x66, pack8, lit, 0xAB, pack8, exit
defword stos32, 0
  dq docol, lit, 0xAB, pack8, exit
defword stos64, 0
  dq docol, rex_w, lit, 0xAB, pack8, exit
defword rep_stos8, 0
  dq docol, lit, 0xF3, pack8, lit, 0xAA, pack8, exit
defword rep_stos16, 0
  dq docol, lit, 0xF3, pack8, lit, 0x66, pack8, lit, 0xAB, pack8, exit
defword rep_stos32, 0
  dq docol, lit, 0xF3, pack8, lit, 0xAB, pack8, exit
defword rep_stos64, 0
  dq docol, lit, 0xF3, pack8, rex_w, lit, 0xAB, pack8, exit

; Stack:
;   output point
defword cmps8, 0
  dq docol, lit, 0xA6, pack8, exit
defword cmps16, 0
  dq docol, lit, 0x66, pack8, lit, 0xA7, pack8, exit
defword cmps32, 0
  dq docol, lit, 0xA7, pack8, exit
defword cmps64, 0
  dq docol, rex_w, lit, 0xA7, pack8, exit
defword repz_cmps8, 0
  dq docol, lit, 0xF3, pack8, lit, 0xA6, pack8, exit
defword repz_cmps16, 0
  dq docol, lit, 0xF3, pack8, lit, 0x66, pack8, lit, 0xA7, pack8, exit
defword repz_cmps32, 0
  dq docol, lit, 0xF3, pack8, lit, 0xA7, pack8, exit
defword repz_cmps64, 0
  dq docol, lit, 0xF3, pack8, rex_w, lit, 0xA7, pack8, exit
defword repnz_cmps8, 0
  dq docol, lit, 0xF2, pack8, lit, 0xA6, pack8, exit
defword repnz_cmps16, 0
  dq docol, lit, 0xF2, pack8, lit, 0x66, pack8, lit, 0xA7, pack8, exit
defword repnz_cmps32, 0
  dq docol, lit, 0xF2, pack8, lit, 0xA7, pack8, exit
defword repnz_cmps64, 0
  dq docol, lit, 0xF2, pack8, rex_w, lit, 0xA7, pack8, exit

; Stack:
;   output point
defword scas8, 0
  dq docol, lit, 0xAE, pack8, exit
defword scas16, 0
  dq docol, lit, 0x66, pack8, lit, 0xAF, pack8, exit
defword scas32, 0
  dq docol, lit, 0xAF, pack8, exit
defword scas64, 0
  dq docol, rex_w, lit, 0xAF, pack8, exit
defword repz_scas8, 0
  dq docol, lit, 0xF3, pack8, lit, 0xAE, pack8, exit
defword repz_scas16, 0
  dq docol, lit, 0xF3, pack8, lit, 0x66, pack8, lit, 0xAF, pack8, exit
defword repz_scas32, 0
  dq docol, lit, 0xF3, pack8, lit, 0xAF, pack8, exit
defword repz_scas64, 0
  dq docol, lit, 0xF3, pack8, rex_w, lit, 0xAF, pack8, exit
defword repnz_scas8, 0
  dq docol, lit, 0xF2, pack8, lit, 0xAE, pack8, exit
defword repnz_scas16, 0
  dq docol, lit, 0xF2, pack8, lit, 0x66, pack8, lit, 0xAF, pack8, exit
defword repnz_scas32, 0
  dq docol, lit, 0xF2, pack8, lit, 0xAF, pack8, exit
defword repnz_scas64, 0
  dq docol, lit, 0xF2, pack8, rex_w, lit, 0xAF, pack8, exit

; Stack:
;   output point
;   source register name
;   target register name
defword add_reg64_reg64, 0
  dq docol
  dq roll3, rex_w, lit, 0x01, pack8, unroll3
  dq swap, reg64, swap, addressing_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   target register name
defword add_indirect_reg64_reg64, 0
  dq docol
  dq roll3, rex_w, lit, 0x01, pack8, unroll3
  dq swap, reg64, swap, addressing_indirect_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   target register name
defword add_reg64_indirect_reg64, 0
  dq docol
  dq roll3, rex_w, lit, 0x03, pack8, unroll3
  dq reg64, swap, addressing_indirect_reg64
  dq exit

; Stack:
;   output point
;   source value
;   target register name
defword add_reg64_imm8, 0
  dq docol
  dq roll3, rex_w, lit, 0x83, pack8, swap, lit, 0, swap, addressing_reg64
  dq swap, pack8
  dq exit

; Stack:
;   output point
;   source register name
;   target register name
defword sub_reg64_reg64, 0
  dq docol
  dq roll3, rex_w, lit, 0x2B, pack8, unroll3
  dq reg64, swap, addressing_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   target register name
defword sub_indirect_reg64_reg64, 0
  dq docol
  dq roll3, rex_w, lit, 0x2B, pack8, unroll3
  dq swap, reg64, swap, addressing_indirect_reg64
  dq exit

; Stack:
;   output point
;   source value
;   target register name
defword sub_reg64_imm8, 0
  dq docol
  dq roll3, rex_w, lit, 0x83, pack8, swap, lit, 5, swap, addressing_reg64
  dq swap, pack8
  dq exit

; Stack:
;   output point
;   source value
;   target register name
defword sbb_reg64_imm8, 0
  dq docol
  dq roll3, rex_w, lit, 0x83, pack8, swap, lit, 3, swap, addressing_reg64
  dq swap, pack8
  dq exit

;   The target register is always rax.
;
; Stack:
;   output point
;   source register name
defword mul_reg64, 0
  dq docol
  dq swap, rex_w, lit, 0xF7, pack8, swap
  dq lit, 4, swap, addressing_reg64
  dq exit

;   The dividend is 128 bits, and is formed from rdx as the high half and rax
; as the low half. The divisor is a specified register. The quotient is
; returned in rax, truncated towards zero. The remainder is in rdx. This
; entire process is unsigned.
;
;   The official mnemonic for this is "div", but divmod is what it does.
;
; Stack:
;   output point
;   divisor register name
defword divmod_reg64, 0
  dq docol
  dq swap, rex_w, lit, 0xF7, pack8, swap
  dq lit, 6, swap, addressing_reg64
  dq exit

; Same as divmod, but signed.
;
; Stack:
;   output point
;   divisor register name
defword idivmod_reg64, 0
  dq docol
  dq swap, rex_w, lit, 0xF7, pack8, swap
  dq lit, 7, swap, addressing_reg64
  dq exit

; Stack:
;   output point
;   target register name
defword inc_reg64, 0
  dq docol
  dq swap, rex_w, lit, 0xFF, pack8, swap, lit, 0, swap, addressing_reg64
  dq exit

; Stack:
;   output point
;   target register name
defword dec_reg64, 0
  dq docol
  dq swap, rex_w, lit, 0xFF, pack8, swap, lit, 1, swap, addressing_reg64
  dq exit

; Stack:
;   output point
;   source register name
;   target register name
defword and_reg64_reg64, 0
  dq docol
  dq roll3, rex_w, lit, 0x23, pack8, unroll3
  dq reg64, swap, addressing_reg64
  dq exit

; Stack:
;   output point
;   source value
;   target register name
defword and_reg64_imm8, 0
  dq docol
  dq roll3, rex_w, lit, 0x83, pack8, swap
  dq lit, 4, swap, addressing_reg64
  dq swap, pack8
  dq exit

; Stack:
;   output point
;   source register name
;   target register name
defword or_reg64_reg64, 0
  dq docol
  dq roll3, rex_w, lit, 0x0B, pack8, unroll3
  dq reg64, swap, addressing_reg64
  dq exit

; Stack:
;   output point
;   source value
;   target register name
defword or_reg64_imm8, 0
  dq docol
  dq roll3, rex_w, lit, 0x83, pack8, swap
  dq lit, 1, swap, addressing_reg64
  dq swap, pack8
  dq exit

; Stack:
;   output point
;   source register name
;   target register name
defword xor_reg64_reg64, 0
  dq docol
  dq roll3, rex_w, lit, 0x33, pack8, unroll3
  dq reg64, swap, addressing_reg64
  dq exit

; Stack:
;   output point
;   target register name
defword not_reg64, 0
  dq docol
  dq swap, rex_w, lit, 0xF7, pack8
  dq swap, lit, 2, swap, addressing_reg64
  dq exit

; Stack:
;   output point
;   left register name
;   right register name
defword cmp_reg64_reg64, 0
  dq docol
  dq roll3, rex_w, lit, 0x3B, pack8, unroll3
  dq reg64, swap, addressing_reg64
  dq exit

; Stack:
;   output point
;   left register name
;   right register name
defword test_reg64_reg64, 0
  dq docol
  dq roll3, rex_w, lit, 0x85, pack8, unroll3
  dq swap, reg64, swap, addressing_reg64
  dq exit

; Stack:
;   output point
;   condition code name
;   target register name
defword set_reg8_cc, 0
  dq docol
  dq roll3, lit, 0x0F, pack8
  dq roll3, conditioncode, lit, 0x90, opcodecc
  dq swap, reg8, lit, 3, lit, 0, roll3, modrm
  dq exit

; Stack:
;   output point
;   address offset value
;   condition code name
defword jmp_cc_rel_imm8, 0
  dq docol
  dq roll3, swap, conditioncode, lit, 0x70, opcodecc
  dq swap, pack8
  dq exit

; Stack:
;   output point
;   address offset value
;   condition code name
defword jmp_cc_rel_imm32, 0
  dq docol
  dq unroll3, lit, 0x0F, pack8
  dq swap, conditioncode, lit, 0x70, opcodecc
  dq swap, pack32
  dq exit

; Stack:
;   output point
;   register name
defword jmp_abs_indirect_reg64, 0
  dq docol
  dq swap, lit, 0xFF, pack8, swap
  dq lit, 4, swap, addressing_indirect_reg64
  dq exit

; Stack:
;   output point
;   address offset value
defword jmp_rel_imm8, 0
  dq docol
  dq swap, lit, 0xEB, pack8
  dq swap, pack8
  dq exit

; Stack:
;   output point
;   address offset value
defword jmp_rel_imm32, 0
  dq docol
  dq swap, lit, 0xE9, pack8
  dq swap, pack32
  dq exit

; Stack:
;   output point
defword syscall, 0
  dq docol, lit, 0x0F, pack8, lit, 0x05, pack8, exit

; Stack:
;   output point
defword hlt, 0
  dq docol, lit, 0xF4, pack8, exit


;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; Runtime word definition ;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;
;;;   We need to be able to use Forth code that defines other Forth code; it
;;; will be the easiest way to create our Forth-flavored assembly languge. To
;;; do that, there's a critical bootstrapping problem: The variables that we
;;; allocate on the heap in _start must be used by the Forth words that define
;;; other Forth words (and, in general, they are needed for anything that
;;; does memory management). Since the heap's address isn't known until
;;; runtime, we can't use build-time labels to locate the heap or the
;;; variables on it, in the way we've done for everything up to now.
;;;
;;;   In fact, it's worse than that: nothing we define at build time can
;;; statically reference anything on the heap; it must always be given the
;;; heap's address as a parameter. That also means statically defined words
;;; can't reference runtime words, at all, except via special code that goes
;;; outside the normal Forth execution model.
;;;
;;;   So, what we'll want to do to deal with that is copy ourselves onto the
;;; heap, and run the rest of the program from there. It would be okay in
;;; principle to keep a few things such as docol in the static code segment,
;;; as long as they don't need to reference the heap, though it might in some
;;; sense be more elegant to entirely abandon the code segment and run
;;; heap-only. At any rate, to keep the set of stuff we have to copy small,
;;; we'll want to make the change-over as soon as possible.
;;;
;;;   Following a common Forth practice, we implement variables as words that
;;; push the corresponding addresses onto the stack. Since the variables that
;;; define the heap are ON the heap, we will definitely need two distinct
;;; versions of these; having the two versions be almost-compatible allows us
;;; to minimize code duplication. Since they do have slightly different
;;; interfaces, we prefix the names of the early ones with "early_".

;   First, we have some words that follow the pack* idioms and are used to
; build specific assembly-based constructs needed in word implementations.
; You can think of them as macros. They are in a sense reimplementations of
; their flatassembler equivalents, which are far, far above, and have more
; documentation.
;
; Stack in:
;   base address
; Stack out:
;   new base address
defword pack_next, 0
  dq docol, lods64, rax, jmp_abs_indirect_reg64, exit

;   This is another helper "macro" that we'll use in defining assembly words
; from Forth. As before, see the flatassembler version for more explanation.
;
; Stack in:
;   base address
;   destination address (absolute)
; Stack out:
;   new base address
defword pack_beforenext, 0
  dq docol, rax, mov_reg64_imm64, rax, jmp_abs_indirect_reg64, exit

;   This is another helper "macro" that we'll use in defining assembly words
; from Forth. In particular, this one is used in docol. As before, see the
; flatassembler version for more explanation.
;
; Stack in:
;   base address
;   source register keyword
; Stack out:
;   new base address
defword pack_pushcontrol, 0
  dq docol
  dq swap, rbp, lit, -8, rbp, lea_reg64_disp8_reg64, swap
  dq rbp, lit, 0, mov_disp8_reg64_reg64
  dq exit

;   This is another helper "macro" that we'll use in defining assembly words
; from Forth. In particular, this one is used in "exit". See the flatassembler
; version for more explanation.
;
; Stack in:
;   base address
;   target register keyword
; Stack out:
;   new base address
defword pack_popcontrol, 0
  dq docol
  dq rbp, lit, 0, roll3, mov_reg64_disp8_reg64
  dq rbp, lit, 8, rbp, lea_reg64_disp8_reg64
  dq exit

;   Now, we have a bunch of words that are used for traversing the Forth
; core data structures that describe words. First, we have a couple that
; relate to individual words and their pieces...
;
;   Jonesforth calls this "TFCA" and ">CFA"; its author speculates that the
; original meaning is "code field address".
defword entry_to_execution_token, 0
  dq docol
  ; Skip next-entry pointer, flag byte, and start terminator.
  dq lit, 10, add
  ; Skip string contents.
  dq dup, stringlen, add
  ; Skip one for the null terminator, seven more for alignment.
  dq lit, 8, add
  ; Zero the low bits and now it's aligned.
  dq lit, 7, invert, and
  dq exit

;   Jonesforth calls this "CFA>". Jonesforth's implementation searches the
; entire dictionary, since its word header format isn't designed to be
; traversed in reverse, but ours is, so it should be fast.
defword execution_token_to_entry, 0
  dq docol
  dq lit, 1, sub
  dq dup, reverse_padding_len, sub
  dq dup, reverse_stringlen, sub
  dq lit, 9, sub
  dq exit

; Stack in:
;   entry address
; Stack out:
;   flags byte
defword fetch_entry_flags, 0
  dq docol
  dq lit, 8, add, fetch, lit, 0xFF, and
  dq exit

; Stack in:
;   entry address
;   flags byte
defword store_entry_flags, 0
  dq docol
  dq swap, lit, 8, add, dup, fetch
  ; (new flags byte, address of flags byte, old flags byte + context)
  dq roll3, lit, 0xFF, and
  dq swap, lit, 0xFFFFFFFFFFFFFF00, and, or
  dq swap, store
  dq exit

defword entry_to_name, 0
  dq docol
  dq lit, 10, add
  dq exit

;   Now, having finished with individual words, we have a bunch of stuff that
; traverses the overall dictionary structure, which is formed by a linked list
; using the pointer that each word starts with.
;
;   This one is the backend for early_find_in; it will eventually also be the
; backend for a non-early version.
;
; Stack in:
;   dictionary to search within
;   name string to find
; Stack out:
;   execution token or 0
defword find_in, 0
  dq docol
  ; It will be more convenient to have the dictionary pointer on top.
  dq swap

  ; If the dictionary pointer is null, exit.
  ; (name string to find, current word pointer)
  dq dup, lit, 0, eq, zbranch, 4*8, swap, drop, exit

  ; Check this entry's "hidden" flag.
  ; (name string to find, current word pointer)
  dq dup, fetch_entry_flags, lit, 0x80, and, lit, 0x80, ne, zbranch, 8*8

  ; Test whether this entry is a match.
  ; (name string to find, current word pointer)
  dq dup2, lit, 10, add, stringcmp, zbranch, 4*8

  ; If we're here, it's not a match; traverse the pointer and repeat.
  ; (name string to find, current word pointer)
  dq fetch, branch, -28*8

  ; If we're here, it's a match. Clean up our working state and exit.
  ; (name string to find, current word pointer)
  dq swap, drop, exit

;   This is the backend for early_next_newer_entry_in; it will eventually also
; be the backend for a non-early version.
;
;   This returns zero if the entry isn't found at all.
;
; Stack in:
;   dictionary to search within
;   entry address
; Stack out:
;   entry address or 0
defword next_newer_entry_in, 0
  dq docol
  ; Dictionary pointer on top
  dq swap
  ; Exit if null
  dq dup, lit, 0, eq, zbranch, 4*8, swap, drop, exit
  ; Test if it's a match
  dq dup2, fetch, ne, zbranch, 4*8
  ; Non-match case; loop
  dq fetch, branch, -16*8
  ; Match case; return
  dq swap, drop, exit

;   This is the backend for early_guess_entry_end; it will eventually also be
; the backend for a non-early version.
;
; Stack in:
;   "here" value
;   dictionary to search within
;   entry address
; Stack out:
;   next entry address or 0
defword guess_entry_end_in, 0
  dq docol
  ; Check whether the entry is flagged as metadata...
  dq dup, fetch_entry_flags, lit, 0x40, and, lit, 0x40, eq, zbranch, 6*8
  ; ... if so, return the same entry address as the result, which means we're
  ; saying the length is zero.
  dq swap, drop, swap, drop, exit
  ; If not, search backwards from the end to find the following entry.
  ; (here, original entry address)
  dq dup, unroll3
  ; (original entry address, here, original entry address)
  dq next_newer_entry_in
  ; (original entry address, here, next entry address or 0)

  ; Check whether we were able to find the next-entry address.
  dq dup, lit, 0, eq, zbranch, 4*8

  ; This is the branch where the original entry is not in the list. Return
  ; the original entry address as the result; as above, this means we're
  ; saying the length is zero. Since we don't have anything else to go on,
  ; this is the safe assumption.
  dq drop, drop, exit

  ; This is the branch where we found it. Return the next entry address.
  dq swap, drop, swap, drop, exit

;   That's it for the code that traverses the dictionary; now we have some
; code meant for use in debugging, which prints out the contents of individual
; words' bodies.
;
;   This one is the backend for early_show_hex; it will eventually also be the
; backend for a non-early version.
;
; Stack in:
;   entry address
;   end address
defword show_hex_between, 0
  dq docol
  dq swap, entry_to_execution_token, lit, 8, add
  ; (end address, current address)
  dq dup2, ge, zbranch, 4*8, drop, drop, exit
  dq litstring, " ", emitstring
  dq dup, fetch, dothex64
  dq lit, 8, add
  dq branch, -17*8

;   This one "decompiles" a Forth word, printing the names of the codewords it
; consists of. It also understands all the built-in literal-related words, and
; serves as an example of how to do that.
;
;   This is the backend for early_show_source; it will eventually also be the
; backend for a non-early version.
;
; Stack in:
;   entry address
;   end address
defword show_source_between, 0
  dq docol

  dq swap, entry_to_execution_token, lit, 8, add
  ; (end address, current address)

  ; This is the start of the loop.
  ;
  ; If it's zero-length, don't try to print its contents.
  dq dup2, ge, zbranch, 4*8, drop, drop, exit

  ; If it's not a word address, the fetch will segfault. We accept this.
  dq dup, fetch, execution_token_to_entry, entry_to_name

  ; (end address, current address, name)

  dq litstring, " ", emitstring
  dq dup, emitstring
  dq swap, lit, 8, add, swap

  dq dup, litstring, "lit", stringcmp, zbranch, 57*8          ; 6 words
  dq dup, litstring, "litpack8", 0, stringcmp, zbranch, 50*8  ; 7 words
  dq dup, litstring, "litpack16", stringcmp, zbranch, 43*8    ; 7 words
  dq dup, litstring, "litpack32", stringcmp, zbranch, 36*8    ; 7 words
  dq dup, litstring, "litpack64", stringcmp, zbranch, 29*8    ; 7 words
  dq dup, litstring, "branch", stringcmp, zbranch, 23*8       ; 6 words
  dq dup, litstring, "zbranch", stringcmp, zbranch, 17*8      ; 6 words
  dq dup, litstring, "0branch", stringcmp, zbranch, 11*8      ; 6 words
  dq dup, litstring, "litstring", stringcmp, zbranch, 16*8    ; 7 words

  ; This is the plain, non-lit branch.
  dq drop, branch, -82*8

  ; This is the lit branch that expects to be followed by a single word.
  dq litstring, " ", emitstring
  dq drop, dup, fetch, dot
  dq lit, 8, add
  dq branch, -94*8

  ; This is the lit branch that expects to be followed by a string. First, we
  ; print the string...
  dq drop
  dq litstring, " ", emitstring
  dq lit, 0x22, fetch_value_stack, emitstring, drop
  dq dup, emitstring
  dq lit, 0x22, fetch_value_stack, emitstring, drop

  ; ... then, we skip the string.
  ;
  ; We always add a word as padding. If the length is a multiple of 8, this is
  ; the zero word following the contents. If it's not, the division rounds
  ; down so the padding word has the tail of the contents in it too.
  dq dup, stringlen, lit, 8, divmod, swap, drop, lit, 1, add
  ; (end address, current address, n words including padding)
  dq lit, 8, mul, add

  dq branch, -126*8

;   This is the backend for early_show_source_or_hex; it will eventually also
; be the backend for a non-early version.
;
; Stack in:
;   entry address
;   end address
defword show_source_or_hex_between, 0
  dq docol

  ; If it's zero-length, don't try to print its contents.
  dq dup2, eq, zbranch, 2*8, exit

  dq swap, dup, unroll3, swap, roll3
  ; (entry address, end address, entry address)
  dq entry_to_execution_token, dup, fetch, lit, 8, sub, eq, zbranch, 3*8
  dq show_hex_between, exit
  dq show_source_between, exit


;;;;;;;;;;;;;;
;;; Lexing ;;;
;;;;;;;;;;;;;;


; In:
;   next input address
; Out:
;   next input address (updated)
;   input byte
defword key, 0
  dq docol
  dq dup, dup, lit, -8, and, fetch, swap, lit, 7, and
  ; (next input address, 64-bit input word, byte index within word)
  dq dup, zbranch, 13*8
  dq lit, 1, sub, swap, lit, 256, divmod, swap, drop, swap, branch, -14*8
  dq drop, lit, 0xFF, and
  dq swap, lit, 1, add, swap
  dq exit



;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; Heap bootstrapping ;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;
;;;   Now, we have a little more work to do before we can bootstrap the heap.
;;; We have all these early_* words which rely on being told explicitly where
;;; the heap is, most of them pertaining to defining words at runtime. Once
;;; the heap exists, these won't be needed, so we never actually copy them to
;;; the heap. To make sure that's easy to keep track of, they're all together
;;; at the end here.
;;;
;;;   The goal is that none of these do any particularly interesting work,
;;; instead deferring to implementations defined in other sections that will
;;; be re-used by the later, fully-bootstrapped stuff. At the moment, some of
;;; them still do interesting things; it's a work in progress.

; Stack in:
;   heap address
; Stack out:
;   heap address
;   requested variable address
defword early_heap, 0
  dq docol, dup, lit, control_stack_size, add, exit
defword early_s0, 0
  dq docol, early_heap, lit, 8, add, exit
defword early_r0, 0
  dq docol, early_heap, lit, 16, add, exit
defword early_latest, 0
  dq docol, early_heap, lit, 24, add, exit
defword early_here, 0
  dq docol, early_heap, lit, 32, add, exit

; Stack in:
;   heap address
;   name string to find
; Stack out:
;   heap address
;   execution token or 0
defword early_find, 0
  dq docol, swap, early_latest, fetch, swap, unroll3, swap, find_in, exit

; Stack in:
;   heap address
;   entry address
; Stack out:
;   heap address
;   entry address or 0
defword early_next_newer_entry, 0
  dq docol
  dq swap, early_latest, fetch, swap, unroll3, swap, next_newer_entry_in
  dq exit

; This doesn't work on the entry at the end of the ELF .text segment. It does
; work on everything else.
; Stack in:
;   heap address
;   entry address
; Stack out:
;   heap address
;   guessed entry end address (first byte that's not part of it)
defword early_guess_entry_end, 0
  dq docol
  dq swap, early_here, fetch, swap, early_latest, fetch, swap
  ; (entry, here, latest, heap)
  dq lit, 4, unroll, roll3
  ; (heap, here, latest, entry)
  dq guess_entry_end_in
  dq exit

; Stack in:
;   heap address
;   entry address
; Stack out:
;   heap address
defword early_show_hex, 0
  dq docol
  dq dup, unroll3, early_guess_entry_end, swap, unroll3
  ; (heap address, entry address, end address)
  dq show_hex_between
  dq exit

; Stack in:
;   heap address
;   entry address
; Stack out:
;   heap address
defword early_show_source, 0
  dq docol
  dq dup, unroll3, early_guess_entry_end, swap, unroll3
  ; (heap address, entry address, end address)
  dq show_source_between
  dq exit

; Stack in:
;   heap address
;   entry address
; Stack out:
;   heap address
defword early_show_source_or_hex, 0
  dq docol
  dq dup, unroll3, early_guess_entry_end, swap, unroll3
  ; (heap address, entry address, end address)
  dq show_source_or_hex_between
  dq exit

; Stack in:
;   heap address
;   entry address
; Stack out:
;   heap address
defword early_describe, 0
  dq docol
  dq litstring, 0x0a, emitstring
  dq dup, dothex64
  dq litstring, 0x0a, emitstring
  dq dup, entry_to_name, emitstring, litstring, ":", emitstring
  dq litstring, 0x0a, emitstring
  dq litstring, " ", emitstring
  dq early_show_source_or_hex
  dq litstring, 0x0a, emitstring
  dq exit

; Stack in:
;   heap address
defword early_describe_all, 0
  dq docol
  dq early_latest
  dq dup, lit, 0, eq, zbranch, 3*8, drop, exit
  ; Start of the loop
  dq fetch
  dq dup, lit, 0, eq, zbranch, 3*8, drop, exit
  dq dup2, early_describe, drop
  dq branch, -13*8
  dq exit

;   Allocate space by incrementing "here", and output a word header in it.
; Also add it to the "latest" linked list. Use zero as the flag values;
; callers that want something else can do that themselves.
;
;   We'll use the pack* words to do this, the way they update the pointer is
; convenient.
;
;   See "Execution model", above, for reminders of the details of this format.
; Create is responsible for everything up to the codeword, not including it.
;
;   This is surprisingly painful to implement, in terms of stack juggling, but
; then there's a sense in which it's the core trick to bootstrapping the heap.
;
; Stack in:
;   heap address
;   name string
; Stack out:
;   heap address
defword early_create, 0
  dq docol
  ; Retrieve the values of "here" and "latest".
  dq swap, early_here, fetch, swap, early_latest, fetch, swap, lit, 4, unroll
  ; Pack the old value of "latest" as the first field of the header, linking
  ; from the newly-defined word to the next-newest word.
  dq pack64
  ; Pack a null byte as the flags.
  dq lit, 0, pack8
  ; Pack another null byte as the start terminator of the name (which is
  ; terminated at both ends).
  dq lit, 0, pack8
  ; Pack the name and its end terminator.
  dq swap, packstring
  ; Alignment before the codeword.
  dq lit, 8, packalign
  ; Retrieve the value of "here" (which still doesn't reflect our additions),
  ; and store it at the address of "latest". It's the start of our
  ; newly-defined word, which makes it the latest word.
  dq swap, early_here, fetch, swap, early_latest, swap, unroll3, store, swap
  ; Retrieve the address of "here" again and store our updated value there.
  dq swap, early_here, swap, unroll3, store
  dq exit

; Stack in:
;   heap address
;   value to append to current word-in-progress
; Stack out:
;   heap address
defword early_comma, 0
  dq docol
  ; Retrieve the value of "here" and pack the value there.
  dq swap, early_here, fetch, swap, unroll3, swap, pack64
  ; Store the updated value of "here".
  dq swap, early_here, swap, unroll3, store
  dq exit

; Stack in:
;   heap address
; Stack out:
;   heap address
defword early_self_codeword, 0
  dq docol, early_here, fetch, lit, 8, add, early_comma, exit

; Stack in:
;   heap address
; Stack out:
;   heap address
defword early_docol_codeword, 0
  dq docol
  dq litstring, "docol", early_find
  dq entry_to_execution_token, execute, early_comma
  dq exit

; Stack in:
;   heap address
;   new value to overwrite "here"
; Stack out:
;   heap address
defword early_here_store, 0
  dq docol, swap, early_here, swap, unroll3, store, exit

; Stack in:
;   heap address
;   address for new variable word to return
;   name string
; Stack out:
;   heap address
defword early_variable, 0
  dq docol
  dq swap, unroll3, early_create, early_self_codeword
  ; (address to return, heap address)
  dq early_here, fetch, swap, unroll3, swap
  ; (heap address, modified "here" value, address to return)
  dq rax, mov_reg64_imm64, rax, push_reg64
  ; (heap address, modified "here" value)
  dq pack_next, lit, 8, packalign, early_here_store
  dq exit

; Stack in:
;   heap address
;   name string
; Stack out:
;   heap address
defword early_keyword, 0
  dq docol
  dq early_create, early_here, fetch, swap
  ; (self codeword address, heap address)
  dq early_docol_codeword
  dq litstring, "lit", early_find, entry_to_execution_token, early_comma
  dq swap, early_comma
  ; (heap address)
  dq litstring, "exit", early_find, entry_to_execution_token, early_comma
  dq early_here, fetch, lit, 8, packalign, early_here_store
  dq exit


;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; Now.... what was our original goal, again? ;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;
;;;   Finally, having fully bootstrapped our runtime environment, we move on
;;; to the core stuff we actually want to accomplish. For this quine, that's
;;; outputting itself.

;;;
;;;   One of the most charming naming traditions in Forth is that the
;;; top-level word that stays running forever, is called "quit".
;;;
defword quit, 0
  dq docol

  ;;;
  ;;; Although we initialized rbp already, we do so again because we'll want
  ;;; that on subsequent visits to this word - it's the main thing it's for.
  ;;; Keep in mind that rsi is the actual "instruction pointer", and we're
  ;;; leaving it unchanged, we just get rid of everything above it.
  ;;;
  ;dq r0, control!                ; overwrite rbp to reset the control stack
    ; TODO though the implementation of r0 is trivial, it depends on where we
    ; put the heap, so it can't be hardcoded, we'll have to build it in RAM.
    ; the same therefore goes for anything that needs to call it. so we can't
    ; call it here, now, yet - we have prep work to do first

  ;;;
  ;;; Do the read-eval-print-loop, which is the main body of the Forth
  ;;; interpreter.
  ;;;
  ;dq interpret                   ; run the repl
  dq quine

  ;;;
  ;;; If the repl ever exits, do it all again.
  ;;;
  ;dq branch, quit - $
  dq lit, 0, sys_exit


;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; (new) Implementation strategy ;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;
;;;   We assemble the entire file contents in a heap-allocated buffer. When
;;; the file is fully assembled, we output it.
;;;
defword quine, 0
  dq docol

  ; We still have "heap" on the stack, so we can call early_here. Add a
  ; constant to "here" in-place, keeping a copy of the pointer ...
  dq early_here, lit, 0x78, swap, addstore
  ; ... and now we have allocated a block of memory, with its address on the
  ; stack. We also still have "heap" at the bottom of the stack, for future
  ; use.

  ; We have one label, and two pieces of information about it: value and
  ; status. We keep them on the stack. We also have a mutable copy of the
  ; buffer's address.
  ;
  ; This stack layout is the interface used throughout the "assembler"; this
  ; quine routine uses it when calling the routines that build the various
  ; pieces of the output, and those pieces in turn use it when calling the
  ; label-handling routines.
  ;
  ; Stack:
  ;   "heap"
  ;   start address of allocated output
  ;   label's value
  ;   label's status
  ;   address of current output point within allocated block
  ; (top)
  ;
  ; Status is a bit field:
  ;    bit zero is whether it was used before being defined
  ;    bit one is whether it's been defined
  ;    bit two is whether the guessed value wound up equaling the actual value
  dq dup, lit, 0, lit, 0, roll3

  ; This takes an address to write to on the stack, writes stuff there, and
  ; returns the next address after where it stopped Writing.
  dq all_contents

  ; The two-pass magick: ; Reset the output offset to the beginning of the
  ; block.
  dq drop, roll3, dup, lit, 4, unroll
  dq all_contents

  ; Drop the copy of the buffer's address.
  dq drop

  ; Drop the label data.
  dq drop, drop

  ; This takes a buffer's address on the stack, skips an ELF file header and
  ; program header based on hardcoded size, computes an offset (secretly
  ; hardcoded), and writes that offset into an appopriate place in the middle
  ; of those headers. It then returns the length of the used portion of the
  ; buffer.
  ;dq lit, 0x78, swap
  dq lit, file_size, swap

  ; write() from stack-allocated buffer
  dq sys_write

  dq exit

; Stack in:
;   output memory start
;   label value (guessed or actual)
;   label status
;   output memory current point
; Stack out:
;   output memory start
;   label value (guessed or actual)
;   label status (potentially modified)
;   output memory current point
;   label value for caller to use
defword use_label, 0
  dq docol

  ; Fetch the status
  dq swap
  ; Check the bit that indicates it's been set.
  dq dup, lit, 2, and, zbranch, 10*8

  ; If we're here, it has been set already, so just put the status back...
  dq swap
  ; Fetch the actual value...
  dq lit, 3, roll, dup, lit, 4, unroll
  ; ... and exit
  dq exit

  ; If we're here, it hasn't been set yet, so mark it used-before-set.
  dq lit, 1, or
  ; Put the status back...
  dq swap
  ; Fetch the guessed value...
  dq lit, 3, roll, dup, lit, 4, unroll
  ; ... and exit
  dq exit

; Stack in:
;   output memory start
;   label value (guessed or actual)
;   label status
;   output memory current point
; Stack out:
;   output memory start
;   label value (now set to actual)
;   label status (modified)
;   output memory current point
defword set_label, 0
  dq docol

  ; Compute the current offset, to use as the actual value
  dq dup, lit, 5, roll, dup, lit, 6, unroll, sub
      ; old value, status, point, new value

  ; Overwrite the old actual value; keep a copy
  dq dup, lit, 5, roll
      ; status, point, new value, new value, old value
  dq eq, swap
      ; status, point, equality, new value
  dq lit, 4, unroll
      ; new value, status, point, equality

  ; We don't need to branch. Now we mark the status as having been defined,
  ; and we also set bit 2 if appropriate.
  dq lit, 4, mul
  dq lit, 3, roll, or, lit, 2, or, swap

  dq exit


; This takes an address to write to on the stack, writes stuff there, and
; returns the next address after where it stopped Writing.
;
; It also makes use of label stuff, further back on the stack.
defword all_contents, 0
  dq docol
  dq elf_file_header, elf_program_header
  dq output_start_routine
  dq self_raw
  dq set_label
  dq exit

;;;
;;; ELF header
;;;
;;;   This is the top-level ELF header, for the entire file. An ELF always has
;;; exactly one of this header, which is always at the start of the file.
;;;
defword elf_file_header, 0
  dq docol

  dq litpack32, 0x7f bappend "ELF"          ; magic number
  dq litpack8, 2                            ; 64-bit
  dq litpack8, 1                            ; little-endian
  dq litpack8, 1                            ; ELF header format v1
  dq litpack8, 0                            ; System-V ABI
  dq litpack64, 0                           ; (padding)

  dq litpack16, 2                           ; executable
  dq litpack16, 0x3e                        ; Intel x86-64
  dq litpack32, 1                           ; ELF format version

  ; Compute the entry pointer.
  dq litpack64, _start                      ; entry point
    ; The offset of _start. This includes the origin, intentionally.

  dq litpack64, 64                          ; program header offset
    ; We place the program header immediately after the ELF header. This
    ; offset is from the start of the file.
  dq litpack64, 0                           ; section header offset
  dq litpack32, 0                           ; processor flags
  dq litpack16, 64                          ; ELF header size
  dq litpack16, 56                          ; program header entry size
  dq litpack16, 1                           ; number of program header entries
  dq litpack16, 0                           ; section header entry size
  dq litpack16, 0                           ; number of section header entries
  dq litpack16, 0                           ; section name string table index

  dq exit


;;;
;;; Program header
;;;
;;;   An ELF program header consists of any number of these entries; they are
;;; always consecutive, but may be anywhere in the file. We always have
;;; exactly one, and it's always right after the ELF file header.
;;;
defword elf_program_header, 0
  dq docol

  dq litpack32, 1                          ; "loadable" segment type
  dq litpack32, 0x05                       ; read+execute permission
  dq litpack64, 0                          ; offset in file
  dq litpack64, $$                         ; virtual address
    ; required, but can be anything, subject to alignment
  dq litpack64, 0                          ; physical address (ignored)

  ; Fill in 0 as the file size for now, to avoid unitialized memory.
  dq use_label, pack64                     ; size in file
  dq use_label, pack64                     ; size in memory

  dq litpack64, 0                          ; segment alignment
    ; for relocation, but this doesn't apply to us

  dq exit


defword output_start_routine, 0
  dq docol
  dq cld
  dq lit, 9, rax, mov_reg64_imm64
  dq lit, heap_requested_address, rdi, mov_reg64_imm64
  dq lit, heap_size, rsi, mov_reg64_imm64
  dq lit, 0x07, rdx, mov_reg64_imm64
  dq lit, 0x22, r10, mov_extrareg64_imm64
  dq lit, 0, r8, mov_extrareg64_imm64
  dq lit, 0, r9, mov_extrareg64_imm64
  dq syscall
  dq rax, rdi, mov_reg64_reg64
  dq exit


; write() the machine code by using self-reference
; TODO do this in a "real" quine way
defword self_raw, 0
  dq docol
  dq dup
  dq lit, elf_header + 0xc4      ; source
  dq lit, file_size - 0xc4       ; length
  ; destination destination source length
  dq dup, lit, 4, roll, add, lit, 4, unroll
  ; result destination source length
  dq memcopy ; broken since memcopy args change
  dq exit


; Routines that traverse the raw code need a way to guess where the end of a
; word's contents are, lest they traverse off the edge of the world and
; segfault. On the heap, we have the "here" pointer to provide an upper bound.
; For code that's loaded from the ELF's text segment, we don't get to do that,
; so we need another way.
;
; We could notionally use syscalls to check how large the text segment is, but
; that would make us more dependent on specifics of the ABI and the kernel.
; Instead, we invent this concept of a metadata word, which is designated with
; a new flag bit and exists to annotate the address space rather than to be
; invoked. The traversal routines hardcode the expectation that the text
; segment ends with a metadata word. Also, they skip metadata words when
; describing things.
;
; Since the boot source is a raw string, not a Forth word, it also needs to
; be flagged as metadata. For convenience's sake, we use a single metadata
; word both to delimit the segment end, and to hold the boot source. Cute,
; right?
defword boot_source, 0x40
  ;   Keep in mind that these words don't exist in memory, so branching won't
  ; have the intended effect. Any logic that requires branching needs to be
  ; written to the heap instead, and branch while running there. In fact,
  ; let's start out with an example of that, which is also our top-level
  ; routine!
  dq ": quit r0 @ control! interpret branch [ -2 8 * , ] ; quit       "
  ;   o/~ Like a whirlpool and it never ends. o/~
  ;
  ;   This implementation of quit does the same thing it does in Jonesforth:
  ; First, it wipes the control stack, completely emptying it so that whatever
  ; calls brought us here are forgotten and no longer relevant to what happens
  ; going forward. Second, it calls interpret repeatedly, forever.
  ;
  ;   Interpret will return control to quit after every iteration. The control
  ; stack doesn't get wiped on those iterations, but user code that wants to
  ; halt its own execution can always call quit to do that, which will once
  ; again wipe the control stack. Isn't that trippy? Instead of unwinding the
  ; stack it just overwrites it!
  ;
  ;   Notice that we immediately call quit, before doing anything else. That's
  ; load-bearing. Keep it that way. To get this far, we used a trampoline,
  ; created in cold_start, that calls interpret in an unrolled loop, just
  ; barely enough times to successfully reach that invocation. If you make
  ; quit longer, make sure to also make the trampoline longer and adjust its
  ; starting location. It's been defined with no extra length, in order to
  ; make sure things will crash in a noticeable way if anyone modifies it
  ; without that follow-through. It would be important to have a close look at
  ; the trampoline setup regardless, because it's allocated in space that this
  ; code could easily overwrite by accident, so it's better to have it fail in
  ; a predictable way than an unpredictable one.

  ;   From here, we can do more or less whatever we want; nearly all the
  ; bootstrapping concerns have been dealt with. We do have to make sure every
  ; line in boot_source that outputs a string outputs one that's an exact
  ; multiple of eight bytes long; any accidental null-padding that
  ; flatassembler inserts will be treated as a string terminator by
  ; attach-string-to-input-buffer.

  ;   In general, we're going to want to be able to go on little excursions
  ; where we define utility words that are only useful for one task, then
  ; deallocate that stuff after we're done with it. We implement "forget",
  ; which removes both dictionary entries and heap allocations for the entry
  ; pointer it's given and everything that came after.
  ;
  ;   The implementation strategy is the same as Jonesforth's version, but
  ; Jonesforth runs in immediate mode and reads a word to operate on, whereas
  ; ours takes an entry pointer and runs in either compiled or immediate
  ; modes.
  ;
  ; In:
  ;   entry pointer
  dq ": forget dup @ latest ! here ! ;                                "

  ;   We'll be defining a lot of immediate words, so we should set up a terse
  ; way to do that.
  dq ": make-immediate latest @ set-word-immediate ;                  "
  dq ": make-hidden latest @ hide-entry ;                             "
  dq ": make-visible latest @ unhide-entry ;                          "

  ;   The word "'" quotes the following word, looking it up and treating it as
  ; a constant. In immediate mode, the constant winds up on the stack; in
  ; compile mode it gets compiled.
  ;
  ;   There are a few possible implementation strategies here. Running as an
  ; immediate word means there's a clear and unambiguous concept of "the
  ; following word", so that's what we do; otherwise we'd have to get clever
  ; about somehow finding out where we were called from. That means we take on
  ; what would otherwise be the interpreter's responsibility, of checking what
  ; mode we're in. Happily, that's easy to do.
  ;
  ;   Though it might be nice to have high-level flow control for this, our
  ; implementation of "if" below relies on "'" several times, whereas "'" only
  ; branches once. So we bootstrap "'" first.
  dq ": ' word value@ find dropstring-with-result                     "
  dq "  interpreter-flags @ 1 & 0branch [ 2 8 * , ] literal           "
  dq "  ; make-immediate                                              "

  ;   Sooner or later we'll want to define recursive words; this one lets us
  ; do that. It compiles into a call to the word that's currently being
  ; defined (strictly speaking, the one whose definition was most recently
  ; begun).
  dq ": recurse latest @ entry-to-execution-token , ; make-immediate  "

  ;   We use a novel suffix-based approach to flow control. We define words
  ; { and } which describe the boundaries of blocks of code, leaving a
  ; description on the value stack, while still compiling the contents
  ; normally.
  ;
  ;   Then follow-up words such as "if" can use that information to slide
  ; the blocks around and insert any needed branches and other logic.
  ;
  ;   After compiling a { ... } block, the stack is (start pointer, length).
  dq ": { here @ ; make-immediate                                     "
  dq ": } dup here @ swap - ; make-immediate                          "

  ; (start pointer, length)
  dq ": if 2dup swap dup 5 8 * + 3roll                                "
  ; (start pointer, length, start pointer, adjusted start pointer, length)
  dq "  memmove                                                       "
  ; (start pointer, length)
  dq "  swap here @ swap here ! swap                                  "
  ; (old here, length)
  dq "  ' lit entry-to-execution-token , 0 ,                          "
  dq "  ' != entry-to-execution-token ,                               "
  ;   The branch length needs to be one word longer than the block length,
  ; because the length field itself is part of the scope of the branch.
  dq "  ' 0branch entry-to-execution-token , dup 8 + ,                "
  ; (old here, length)
  dq "  drop 5 8 * + here ! ; make-immediate                          "

  ; (start pointer, length)
  dq ": unless 2dup swap dup 5 8 * + 3roll                            "
  ; (start pointer, length, start pointer, adjusted start pointer, length)
  dq "  memmove                                                       "
  ; (start pointer, length)
  dq "  swap here @ swap here ! swap                                  "
  ; (old here, length)
  dq "  ' lit entry-to-execution-token , 0 ,                          "
  dq "  ' = entry-to-execution-token ,                                "
  ;   The branch length needs to be one word longer than the block length,
  ; because the length field itself is part of the scope of the branch.
  dq "  ' 0branch entry-to-execution-token , dup 8 + ,                "
  ; (old here, length)
  dq "  drop 5 8 * + here ! ; make-immediate                          "

  ; (true start, true length, false start, false length)
  dq ": if-else                                                       "
  ;dq "  dup 4 roll dup 5 unroll +                                     "
  ;
  ;   First we slide the false-block forward, then the true-block. We slide
  ; them both directly into their final positions, leaving space at the start
  ; for a test and branch, and space in between for an unconditional branch.
  ; Those spaces will take five words, and two words, respectively. So the
  ; false-block gets moved by seven words, and the true-block gets moved by
  ; five words.
  ;
  ;   Note that this has diverged slightly from the self-hosted version,
  ; because in this version we don't have "pick" yet.
  dq "  2dup swap dup 7 8 * + 3roll memmove                           "
  dq "  4 roll dup 5 unroll 4 roll dup 5 unroll                       "
  dq "  swap dup 5 8 * + 3roll memmove                                "

  ; (true start, true length, false start, false length)
  ;
  ;   Now we write out the initial test-and-branch.
  dq "  4 roll dup 5 unroll here @ 6 unroll here !                    "
  ; (old here, true start, true length, false start, false length)
  dq "  ' lit entry-to-execution-token , 0 ,                          "
  dq "  ' != entry-to-execution-token ,                               "
  ;   Branch past the length field, the true-block, and the unconditional
  ; branch in the middle.
  dq "  ' 0branch entry-to-execution-token ,                          "
  dq "  3roll dup 4 unroll 3 8 * + ,                                  "
  ;
  ;  Next, write out the unconditional branch in the middle.
  dq "  swap dup 3unroll 5 8 * + here !                               "
  dq "  ' branch entry-to-execution-token ,                           "
  ;  Branch past the length field and the false-block.
  dq "  dup 8 + ,                                                     "
  ;
  ;  Set "here" to point to the true end.
  dq "  drop drop drop drop 7 8 * + here !                            "
  dq "  ; make-immediate                                              "

  ; (start, length)
  dq ": forever                                                       "
  dq "  ' branch entry-to-execution-token , 8 + -1 * , drop           "
  dq "  ; make-immediate                                              "

  ;   This slides the body forward, leaving the test where it is. It puts a
  ; conditional branch in-between them, then appends an unconditional branch
  ; at the end.
  ;
  ; (test start, test length, body start, body length)
  dq ": while                                                         "
  ;   The conditional branch needs five words.
  dq "  2dup swap dup 5 8 * + 3roll memmove                           "
  dq "  here @ 5 unroll swap dup 3unroll here !                       "
  ; (old here, test start, test length, body start, body length)
  dq "  ' lit entry-to-execution-token , 0 ,                          "
  dq "  ' != entry-to-execution-token ,                               "
  ; Branch past the length field, the body, and the unconditional branch.
  dq "  ' 0branch entry-to-execution-token ,                          "
  dq "  dup 3 8 * + ,                                                 "
  ; Set "here" to the new end.
  dq "  5 8 * 6 roll + here !                                         "
  ; (test start, test length, body start, body length)
  ;   Unconditionally branch backwards past the branch word, the body, the
  ; conditional branch, and the test.
  dq "  ' branch entry-to-execution-token ,                           "
  dq "  6 8 * + swap drop + swap drop -1 * ,                          "
  dq "  ; make-immediate                                              "

  ;   This use of bitwise and is okay because they're both either 0 or 1.
  ; We'll have logical and real soon now, be patient... :)
  dq ": is-in-heap dup log @ <= swap here @ > & ;                     "

;  dq ": unlink-pre-heap-words                                         "
;  dq "  latest @                                                      "
;  dq "  dup { dup is-in-heap 0 = { drop exit } if }                   "
;  dq "      { drop exit } if-else                                     "
;  dq "  { dup @ is-in-heap } { @ } while                              "
;  dq "  0 swap ! ;                                                    "
  ; Do it immediately, so that we don't accidentally rely on any of that.
  ;dq "unlink-pre-heap-words                                           "

  ; Now some fancier stack combinators.
  ;
  ;   While it might be nice, for performance reasons, to do these in
  ; assembler, for now it's more important to have them at all.
  dq ": 1- 1 - ;                                                      "
  dq ": 1+ 1 + ;                                                      "
  dq ": max 2dup >= { swap drop } { drop } if-else ;                  "
  dq ": min 2dup <= { swap drop } { drop } if-else ;                  "

  dq ": over swap dup 3unroll ;                                       "
  dq ": pick 2 + dup roll dup 3roll unroll ;                          "
  ; Standard Forth doesn't have equivalents of our ndrop and ndup. The HP
  ; calls them DROPN and DUPN but that doesn't go well with ie. 2dup or 3roll,
  ; so we do it like this.
  dq ": ndrop { dup } { swap drop 1- } while drop ;                   "
  dq ": ndup dup 1+ swap { dup }                                      "
  dq "  { swap dup pick 3unroll swap 1- } while 2drop ;               "
  dq ": 3drop drop drop drop ;                                        "
  dq ": 3dup 2 pick 2 pick 2 pick ;                                   "

  dq ": && 0 != swap 0 != * ;                                         "
  dq ": || | 0 != ;                                                   "
  dq ": not 0 = ;                                                     "
  dq ": negate -1 * ;                                                 "

  dq ": align-floor dup 3unroll /% swap drop * ;                      "

  dq ": nexit 8 * control@ + r0 @ min control! ;                      "

  ; Let's have strings now.
  dq ": s""                                                            "
  dq "  consume                                                       "
  dq "  interpreter-flags @ 1 &                                       "
  dq "    { ' litstring entry-to-execution-token , } if               "
  dq "  here @ key { dup dup 34 != && } { pack8 key } while           "
  dq "  drop 0 pack8                                                  "
  dq "  interpreter-flags @ 1 &                                       "
  dq "    { 8 packalign here ! }                                      "
  dq "    { drop here @ } if-else                                     "
  dq "  ; make-immediate                                              "

  ; Finicky semantics, but also important.
  dq ": .""                                                            "
  dq "  ' s"" entry-to-execution-token execute                         "
  dq "  interpreter-flags @ 1 &                                       "
  dq "    { ' emitstring entry-to-execution-token , }                 "
  dq "    { emitstring } if-else                                      "
  dq "  ; make-immediate                                              "

  ; While we're thinking about input, let's also have comments.
  dq ": ~                                                             "
  dq "  key { dup dup 10 != && } { drop key } while drop              "
  dq "  ; make-immediate                                              "

  ; Some output formatting tools.
  dq ": space 32 value@ emitstring drop ;                             "
  dq ": newline 10 value@ emitstring drop ;                           "
  dq ": indent { dup } { space 1- } while drop ;                      "
  dq ": .hexn 16 swap .base-unsigned ;                                "

  ; Some debugging tools.
  dq ": stack                                                         "
  dq "  s0 @ 8 - { dup value@ 8 + != }                                "
  dq "  { dup s0 @ 8 - != { space } if dup @ . 8 - }                  "
  dq "  while drop newline ;                                          "
  dq ": stackhex                                                      "
  dq "  s0 @ 8 - { dup value@ 8 + != }                                "
  dq "  { dup s0 @ 8 - != { space } if dup @ .hex64 8 - }             "
  dq "  while drop newline ;                                          "

  dq ": oldest-entry                                                  "
  dq "  latest @ { dup @ } { @ } while ;                              "

  dq ": oldest-entry-in                                               "
  dq "  dup                                                           "
  dq "  dup { { dup @ } { @ } while } if                              "
  dq "  dup 3roll = { drop 0 } if ;                                   "

  dq ": next-newer-entry                                              "
  dq "  latest @                                                      "
  dq "  2dup = { 2drop 0 exit } if                                    "
  dq "  { dup { 2dup @ != } if }                                      "
  dq "  { @ } while swap drop ;                                       "

  dq ": guess-entry-end                                               "
  dq "  dup entry-flags@ 64 & 64 = { exit } if                        "
  dq "  dup next-newer-entry dup                                      "
  dq "    { drop dup is-in-heap { drop here @ } { drop } if-else      "
  dq "      exit } unless                                             "
  dq "  swap drop ;                                                   "

  dq ": containing-entry                                              "
  dq "  dup is-in-heap { drop 0 exit } unless                         "
  dq "  latest @ { dup { 2dup > } { 0 } if-else }                     "
  dq "    { @ } while swap drop ;                                     "

  dq ": is-assembly-word                                              "
  dq "  entry-to-execution-token dup 8 + swap @ = ;                   "

  dq ": is-docol-itself                                               "
  dq "  entry-to-name s"" docol"" stringcmp 0 = ;                       "


  ;   If the buffer is empty, make sure its logical position is at the
  ; physical start. Otherwise, leave it alone.
  ;
  ; In:
  ;   pointer to buffer metadata
  dq ": normalize-buffer                                              "
  dq "  dup buffer-logical-length @ 0 =                               "
  dq "  { dup buffer-physical-start @ swap                            "
  dq "    buffer-logical-start ! } { drop } if-else                   "
  dq "  ;                                                             "


  ;   Find the next contiguous area of a buffer to write incoming data to.
  ; This will begin at the logical end. Depending on the order things are in,
  ; the available space might run from there to the physical end, or to the
  ; logical start. Those are the only two possibilities.
  ;
  ;   If the buffer is full, this will return as normal but the length will
  ; be zero. The caller should make sure to respect that.
  ;
  ; In:
  ;   pointer to buffer metadata
  ; Out:
  ;   destination start
  ;   destination length
  dq ": compute-next-buffer-free-block                                "
  dq "  dup buffer-physical-start @ swap dup buffer-physical-length @ "
  dq "  swap 3unroll + swap                                           "
  ; (physical end, metadata pointer)
  dq "  dup buffer-logical-start @ swap dup buffer-logical-length @   "
  dq "  swap 3unroll + swap 3unroll                                   "
  ; (metadata pointer, physical end, logical end not yet wrapped)

  dq "  2dup >=                                                       "
  dq "  {                                                             "
  ;   If the logical end is greater than or equal to the physical end, find
  ; where it wraps to and start from there.
  dq "    swap - swap dup buffer-physical-start @ swap 3unroll +      "
  ; (metadata pointer, destination start)
  ;   In this scenario, the logical start is the end of the free space, so
  ; compute how far away it is.
  dq "    dup 3roll                                                   "
  ; (destination start, destination start, metadata pointer)
  dq "    buffer-logical-start @ swap - }                             "

  dq "  {                                                             "
  dq "    3roll drop                                                  "
  dq "    dup 3roll swap -                                            "
  ;   If the logical end is less than the physical end, it is the destination;
  ; the physical end is also the end of the free space, so compute how far
  ; away it is.
  dq "    } if-else                                                   "
  ; (destination start, destination length)
  dq "  ;                                                             "



  ; In:
  ;   pointer to buffer metadata
  dq ": refill-input-buffer-from-stdin                                "
  dq "  dup normalize-buffer                                          "
  dq "  dup compute-next-buffer-free-block                            "
  ;   Check whether the buffer is full. If not, do a read. If so, that's not
  ; an error, just clean up and take no action.
  dq "  dup { swap sys-read                                           "
  dq "        dup -2 =                                                "
  dq "        { drop drop s"" Read error."" emitstring 0 sys-exit }     "
  dq "        { swap dup buffer-logical-length @ 3roll +              "
  dq "          swap buffer-logical-length ! } if-else }              "
  dq "      { drop drop } if-else ;                                   "


  ;   Though we only do it once, this is a bit involved so we provide it as a
  ; word. Notionally there are situations in which it could come up again, so
  ; it seems worth having around.
  ;
  ;   It actually needs to run as a compiled word no matter what; if it were
  ; run in interpret mode it would cut itself off from the rest of itself.
  ; However, if we didn't want to keep it around we could have it forget
  ; itself...
  dq ": relink-main-input-buffer-to-stdin                             "
  dq "  1024 allocate dup main-input-buffer buffer-physical-start !   "
  dq "  main-input-buffer buffer-logical-start !                      "
  dq "  1024 main-input-buffer buffer-physical-length !               "
  dq "  0 main-input-buffer buffer-logical-length !                   "
  dq "  ' refill-input-buffer-from-stdin entry-to-execution-token     "
  dq "  main-input-buffer input-buffer-refill !                       "
  dq "  ;                                                             "


  ;   Notionally, it might make sense to define "create" in terms of
  ; "create-in". Any change like that is being postponed to after the removal
  ; of flatassembler, when refactorings will be easier.
  ;
  ; In:
  ;   name string pointer
  ;   dictionary handle (points to a pointer to the first item)
  dq ": create-in                                                     "
  dq "  dup @ here @ swap pack64                                      "
  ; (name string pointer, dictionary handle, output point)
  dq "  0 pack8 0 pack8                                               "
  dq "  3roll packstring                                              "
  dq "  8 packalign                                                   "
  ; (dictionary handle, output point)
  dq "  swap here @ swap !                                            "
  ; (output point)
  dq "  here !                                                        "
  dq "  ;                                                             "


  ;   The word named "docol" has the job of returning the value that gets used
  ; as the actual codeword. We make the assumption that the codeword will
  ; point somewhere near the entry header; we allow for the possibility that
  ; it might be before or after.
  ;
  ;   Generally, it's possible for there to be several copies of docol due to
  ; alternate heaps and things like that, so the goal is to recognize any of
  ; them.
  dq ": is-docol-codeword                                             "
  dq "  dup is-in-heap { drop 0 exit } unless                         "
  dq "  containing-entry dup                                          "
  dq "    { dup is-docol-itself                                       "
  dq "      { drop 1 }                                                "
  dq "      { next-newer-entry dup { is-docol-itself } if } if-else   "
  dq "    } if ;                                                      "

  ; TODO this only works on heap words
  dq ": is-docol-word                                                 "
  dq "  dup is-assembly-word { drop 0 exit } if                       "
  dq "  entry-to-execution-token @ is-docol-codeword ;                "

  dq ": word-heading                                                  "
  dq "  dup entry-to-name dup emitstring space                        "
  dq "  stringlen 1+ 54 swap - 0 max indent dup .hex64                "
  dq "  dup entry-flags@ dup                                          "
  dq "    { space                                                     "
  dq "      dup 128 & { s"" H"" emitstring } if                         "
  dq "      dup 64 & { s"" M"" emitstring } if                          "
  dq "      dup 1 & { s"" I"" emitstring } if                           "
  dq "    } if drop                                                   "
  dq "  dup is-assembly-word { s""  asm"" emitstring }                  "
  dq "    { dup is-docol-word { s""  raw"" emitstring } unless          "
  dq "    } if-else drop                                              "
  dq "  newline ;                                                     "

  dq ": list-dictionary                                               "
  dq "  oldest-entry { dup }                                          "
  dq "  { dup word-heading next-newer-entry } while drop ;            "

  ; (content end, content start, label start)
  dq ": hexdump-row                                                   "
  dq "  2 indent dup .hex32 dup 4 unroll                              "
  dq "  0 { dup 16 > }                                                "
  dq "  { dup 7 & 0 = { space } if space                              "
  dq "    2dup + dup 4 pick <= swap 5 pick > &&                       "
  dq "      { 2dup + 8@ .hex8 } { space space } if-else               "
  dq "    1+ } while                                                  "
  dq "  newline 5 ndrop ;                                             "

  ; (end, start)
  dq ": hexdump-between                                               "
  dq "  dup 16 1- invert &                                            "
  dq "  { dup 3 pick > }                                              "
  dq "  { 3dup hexdump-row 16 + } while 3 ndrop ;                     "

  ; (start, length)
  dq ": hexdump-from swap dup 3unroll + swap hexdump-between ;        "

  ; (start)
  dq ": hexdump 64 hexdump-from ;                                     "

  dq ": describe-hex                                                  "
  dq "  dup word-heading                                              "
  dq "  dup guess-entry-end swap entry-to-execution-token             "
  dq "  hexdump-between ;                                             "

  dq ": is-codeword                                                   "
  dq "  dup is-in-heap { drop 0 exit } unless                         "
  dq "  dup containing-entry dup { 2drop 0 exit } unless              "
  dq "  entry-to-execution-token = ;                                  "

  dq ": describe-docol                                                "
  dq "  dup word-heading                                              "
  dq "  dup guess-entry-end swap entry-to-execution-token 8 +         "
  dq "  { 2dup < }                                                    "
  dq "  { space dup @ dup is-codeword                                 "
  dq "    { execution-token-to-entry entry-to-name emitstring }       "
  dq "    { . } if-else                                               "
  dq "    8 + } while newline ;                                       "

  dq ": describe                                                      "
  dq "  dup is-docol-word                                             "
  dq "  { describe-docol } { describe-hex } if-else ;                 "

  dq ": describe-all                                                  "
  dq "  oldest-entry { dup }                                          "
  dq "  { dup describe next-newer-entry } while drop ;                "

  dq ": bye 0 sys-exit ;                                              "

  ;dq ": foo 5 { dup }                                                 "
  ;dq "  { 4 indent 97 value@ emitstring drop newline 1- }             "
  ;dq "  while drop ; foo                                              "
  ;dq ": foo 5 6 > { 42 . } if ; foo ' foo forget                      "
  ;dq ": foo 5 6 > { 42 } { 69 } if-else . ; foo ' foo forget          "
  ;dq ": foo { 5 . } forever ; foo ' foo forget                        "
  ;dq ": foo 0 { dup 5 > } { dup . 1+ } while drop ; foo ' foo forget  "

  ;dq "s"" foo"" value@ emitstring dropstring                            "
  ;dq ": foo s"" zotzotza"" emitstring ; ' foo describe                  "
  ;dq ": foo 1 { s"" zotzotza"" emitstring } if ; ' foo describe         "
  ;dq "s"" zotzotza"" emitstring stack                                   "
  ;dq "68719476736     containing-entry stackhex                       "
  ;dq "68719587456     containing-entry stackhex                       "
  ;dq "587456     containing-entry stackhex                            "
  ;dq "68719607808 containing-entry stackhex                           "
  ;dq "list-dictionary                                                 "
  ;dq "describe-all                                                    "
  ;dq ": doittoit 8    allocate-input-buffer                           "
  ;dq "  dup stackhex refill-input-buffer-from-stdin                   "
  ;dq "  dup key-from drop                                             "
  ;dq "  dup key-from drop                                             "
  ;dq "  dup key-from drop                                             "
  ;dq "  dup key-from drop                                             "
  ;dq "  stackhex dup 256 hexdump-from                                 "
  ;dq "  dup refill-input-buffer-from-stdin                            "
  ;dq "  stackhex dup 256 hexdump-from                                 "
  ;dq "  dup refill-input-buffer-from-stdin                            "
  ;dq "  stackhex dup 256 hexdump-from                                 "
  ;dq "  dup refill-input-buffer-from-stdin                            "
  ;dq "  stackhex dup 256 hexdump-from                                 "
  ;dq "  bye ; doittoit                                                "
  dq "                                                                "

  ; TODO running "variable" interactively doesn't work because the string and
  ; the "create" invocation step on each other. this is a big deal and really
  ; needs to be fixed.
  ; TODO find_in/find-in is backwards compared to create-in.
  ; TODO define ( ... ) comments
  ; TODO define constant (double-check variable)
  ; TODO consider defining is-hidden and is-immediate
  ; TODO define ?
  ; TODO define allot and cells
  ; TODO consider what text stuff to define
  ; TODO control stack trace
  ; TODO make an interactive debugger
  ; TODO argc argv envp etc
  ; TODO consider "unused" and some replacement for "brk"
  ; TODO consider file API
  ; TODO consider ";asm" or something
  ; TODO consider a welcome message

  ; The last in a long series of perilous handoffs. :)
  dq "  relink-main-input-buffer-to-stdin                             "
  ; We'll never get here, but if we do, it's an error.
  dq "  1 sys-exit                                                    "

  dq 0


final_word_name = latest_word
code_size = $ - code_start
file_size = $ - $$