add support for evaluating expressions

This change allows immediate values to be calculated from an expression,
such as 1+1, even including symbols such as "100 << const" (where const
was defined with the .set directive). Expressions are also supported in
the .set directives.

Author: wnienhaus

esp32_ulp/assemble.py

@@ -240,7 +240,7 @@ def d_align(self, align=4, fill=None):
             self.fill(self.section, amount, fill)
 
     def d_set(self, symbol, expr):
-        value = int(expr)  # TODO: support more than just integers
+        value = int(opcodes.eval_arg(expr))  # TODO: support more than just integers
         self.symbols.set_sym(symbol, ABS, None, value)
 
     def d_global(self, symbol):

esp32_ulp/opcodes.py

@@ -6,6 +6,7 @@
 from uctypes import struct, addressof, LITTLE_ENDIAN, UINT32, BFUINT32, BF_POS, BF_LEN
 
 from .soc import *
+from .util import split_tokens
 
 # XXX dirty hack: use a global for the symbol table
 symbols = None
@@ -267,6 +268,22 @@ def make_ins(layout):
 ARG = namedtuple('ARG', ('type', 'value', 'raw'))
 
 
+def eval_arg(arg):
+    parts = []
+    for token in split_tokens(arg):
+        if symbols.has_sym(token):
+            _, _, sym_addr = symbols.get_sym(token)
+            parts.append(str(sym_addr))
+        else:
+            parts.append(token)
+    parts = "".join(parts)
+    try:
+        parts = str(eval(parts))
+    except:
+        pass
+    return parts
+
+
 def arg_qualify(arg):
     """
     look at arg and qualify its type:
@@ -289,8 +306,12 @@ def arg_qualify(arg):
         return ARG(IMM, int(arg), arg)
     except ValueError:
         pass
-    entry = symbols.get_sym(arg)
-    return ARG(SYM, entry, arg)
+    try:
+        entry = symbols.get_sym(arg)
+        return ARG(SYM, entry, arg)
+    except KeyError:
+        pass
+    return ARG(IMM, int(eval_arg(arg)), arg)
 
 
 def get_reg(arg):

esp32_ulp/util.py

@@ -2,10 +2,43 @@
 
 import gc
 
+NORMAL, WHITESPACE = 0, 1
+
 
 def garbage_collect(msg, verbose=DEBUG):
     free_before = gc.mem_free()
     gc.collect()
     free_after = gc.mem_free()
     if verbose:
         print("%s: %d --gc--> %d bytes free" % (msg, free_before, free_after))
+
+
+def split_tokens(line):
+    buf = ""
+    tokens = []
+    state = NORMAL
+    for c in line:
+        if ('a' <= c <= 'z') or ('A' <= c <= 'Z') or ('0' <= c <= '9') or c == '_':
+            if state != NORMAL:
+                if len(buf) > 0:
+                    tokens.append(buf)
+                    buf = ""
+                state = NORMAL
+            buf += c
+        elif c == ' ' or c == '\t':
+            if state != WHITESPACE:
+                if len(buf) > 0:
+                    tokens.append(buf)
+                    buf = ""
+                state = WHITESPACE
+            buf += c
+        else:
+            if len(buf) > 0:
+                tokens.append(buf)
+                buf = ""
+            tokens.append(c)
+
+    if len(buf) > 0:
+        tokens.append(buf)
+
+    return tokens

tests/00_unit_tests.sh

@@ -4,7 +4,7 @@
 
 set -e
 
-for file in opcodes assemble link ; do
+for file in opcodes assemble link util; do
     echo testing $file...
     micropython $file.py
 done

tests/assemble.py

@@ -135,6 +135,28 @@ def test_assemble_uppercase_opcode():
     assert not raised
 
 
+def test_assemble_evalulate_expressions():
+    src_w_expr = """\
+    .set shft, 2
+    .set loops, (1 << shft)
+
+entry:
+    move r0, 1+1
+    move r1, loops
+    move r2, (shft + 10) * 2
+    move r3, entry << 2
+"""
+    a = Assembler()
+    a.assemble(src_w_expr)
+
+    assert a.symbols.has_sym('shft')
+    assert a.symbols.has_sym('loops')
+    assert a.symbols.has_sym('entry')
+    assert a.symbols.get_sym('shft') == (ABS, None, 2)
+    assert a.symbols.get_sym('loops') == (ABS, None, 4)
+    assert a.symbols.get_sym('entry') == (REL, TEXT, 0)
+
+
 def test_symbols():
     st = SymbolTable({}, {}, {})
     for entry in [
@@ -195,4 +217,5 @@ def test_symbols():
 test_assemble_bss_with_value()
 test_assemble_global()
 test_assemble_uppercase_opcode()
+test_assemble_evalulate_expressions()
 test_symbols()

tests/opcodes.py

@@ -1,6 +1,8 @@
 from uctypes import UINT32, BFUINT32, BF_POS, BF_LEN
 from esp32_ulp.opcodes import make_ins, make_ins_struct_def
-from esp32_ulp.opcodes import get_reg, get_imm, get_cond, arg_qualify, ARG, REG, IMM, COND
+from esp32_ulp.opcodes import get_reg, get_imm, get_cond, arg_qualify, ARG, REG, IMM, SYM, COND
+from esp32_ulp.assemble import SymbolTable, ABS, REL, TEXT
+import esp32_ulp.opcodes as opcodes
 
 OPCODE_DELAY = 4
 LAYOUT_DELAY = """
@@ -43,6 +45,19 @@ def test_arg_qualify():
     assert arg_qualify('Eq') == ARG(COND, 'eq', 'Eq')
     assert arg_qualify('EQ') == ARG(COND, 'eq', 'EQ')
 
+    # for the next tests, ensure the opcodes module has a SymbolTable
+    opcodes.symbols = SymbolTable({}, {}, {})
+    opcodes.symbols.set_sym('const', ABS, None, 42)  # constant as defined by .set
+    opcodes.symbols.set_sym('entry', REL, TEXT, 4)  # label pointing to code
+
+    assert arg_qualify('1+1') == ARG(IMM, 2, '1+1')
+    assert arg_qualify('const >> 1') == ARG(IMM, 21, 'const >> 1')
+    assert arg_qualify('entry') == ARG(SYM, (REL, TEXT, 4), 'entry')  # symbols should not (yet) be evaluated
+    assert arg_qualify('entry + const') == ARG(IMM, 46, 'entry + const')
+
+    # clean up
+    opcodes.symbols = None
+
 
 def test_get_reg():
     assert get_reg('r0') == 0

tests/util.py

@@ -0,0 +1,36 @@
+from esp32_ulp.util import split_tokens
+
+tests = []
+
+
+def test(param):
+    """
+    the @test decorator
+    """
+    tests.append(param)
+
+
+@test
+def test_split_tokens():
+    assert split_tokens("") == []
+    assert split_tokens("t") == ['t']
+    assert split_tokens("test") == ['test']
+    assert split_tokens("t t") == ['t', ' ', 't']
+    assert split_tokens("t,t") == ['t', ',', 't']
+    assert split_tokens("test(arg)") == ['test', '(', 'arg', ')']
+    assert split_tokens("test(arg,arg2)") == ['test', '(', 'arg', ',', 'arg2', ')']
+    assert split_tokens("test(arg,arg2)") == ['test', '(', 'arg', ',', 'arg2', ')']
+    assert split_tokens("  test(  arg,  arg2)") == ['  ', 'test', '(', '  ', 'arg', ',', '  ', 'arg2', ')']
+    assert split_tokens("  test(  arg )  ") == ['  ', 'test', '(', '  ', 'arg', ' ', ')', '  ']
+    assert split_tokens("\t  test  \t  ") == ['\t  ', 'test', "  \t  "]
+    assert split_tokens("test\nrow2") == ['test', "\n", "row2"]
+
+    # split_token does not support comments. should generally only be used after comments are already stripped
+    assert split_tokens("test(arg /*comment*/)") == ['test', '(', 'arg', ' ', '/', '*', 'comment', '*', '/', ')']
+    assert split_tokens("#test") == ['#', 'test']
+
+
+if __name__ == '__main__':
+    # run all methods marked with @test
+    for t in tests:
+        t()