Merge pull request #22 from misprit7/terracc

TerraCC

.github/workflows/in-game-tests.yml

@@ -30,7 +30,7 @@ jobs:
         submodules: recursive
 
     - name: Install tinterface
-      run: pip3 install -e ./tinterface
+      run: pip3 install --break-system-packages -e ./tinterface
 
     - name: Run riscof tests
       run: ./run-tests.sh

.gitignore

@@ -4,6 +4,9 @@
 *.wld.bak
 *.wld.bak2
 
+# Me being dumb
+--to
+
 # Docker secrets
 docker/.env
 

docker/Dockerfile

@@ -14,7 +14,7 @@
 ################################################################################
 # Probably should have used this in hindsight:
 # https://github.com/tModLoader/tModLoader/tree/1.4/patches/tModLoader/Terraria/release_extras/DedicatedServerUtils
-FROM steamcmd/steamcmd:latest
+FROM steamcmd/steamcmd@sha256:0d3d8716fb0ac76f02bd9d850334b892d1caf92632fcce362bb52592d63211c3
 
 
 ################################################################################
@@ -27,13 +27,15 @@ RUN apt-get update && apt-get install -y \
     curl \
     gcc \
     git \
-    libicu70 \
+    libsdl2-2.0-0 \
     python3 \
     python3-pip \
     make \
     vim \
     wget
 
+# libicu70 \
+
 
 ################################################################################
 # tModLoader
@@ -42,9 +44,9 @@ RUN --mount=type=secret,id=_env,dst=/etc/secrets/.env \
     echo exit | steamcmd \
         "+login $(sed -n 1p /etc/secrets/.env) $(sed -n 2p /etc/secrets/.env)" \
         "+app_update 1281930 validate" &&\
-    ln -s /root/Steam/steamapps/ /root/.local/share/Steam/ &&\
     mkdir -p /root/.local/share/Terraria/tModLoader/ModSources &&\
     mkdir -p /root/.local/share/Terraria/tModLoader/Mods
+# ln -s /root/Steam/steamapps/ /root/.local/share/Steam/ &&\
 COPY tModLoader.targets /root/.local/share/Terraria/tModLoader/ModSources
 
 
@@ -79,14 +81,15 @@ ENV PATH "$PATH:/opt/riscv/bin"
 # Dotnet
 ################################################################################
 RUN wget https://dot.net/v1/dotnet-install.sh &&\
-    bash ./dotnet-install.sh --install-dir /usr/local/bin -channel STS -version 6.0.100 &&\
+    bash ./dotnet-install.sh --install-dir /usr/local/bin -channel 8.0 &&\
     dotnet --list-sdks
 
 
 ################################################################################
 # Riscof
 ################################################################################
-RUN pip3 install git+https://github.com/riscv/riscof.git
+# RUN pip3 install git+https://github.com/riscv/riscof.git
+RUN pip3 install --break-system-packages git+https://github.com/riscv/riscof.git
 # I should set this up to be built, right now it relies on host machine
 # already having it installed
 # This would require setting up ocaml as well as other build dependencies

test/computerraria/riscof_computerraria.py

@@ -61,7 +61,7 @@ def build(self, isa_yaml, platform_yaml):
   def runTests(self, testList):
 
     logger.info('Starting server')
-    tserver = tinterface.TServer()
+    tserver = tinterface.TServer(terracc=False)
     tserver.start()
     logger.info('Server loaded')
     for testname in testList:

tinterface/tinterface/__init__.py

@@ -2,6 +2,8 @@
 import os, shutil, time, sys
 from typing import Tuple
 import pexpect
+from datetime import datetime
+from tqdm import tqdm
 
 # CI image is under root user
 TMODLOADER_DIR = str(Path('/root/.local/share/Steam/steamapps/common/tModLoader/')) + '/'
@@ -77,12 +79,16 @@ def __init__(
         port=7777,
         inplace=False,
         verbose=False,
+        terracc=False,
+        lazy=False,
     ):
         self.path = TMODLOADER_DIR + 'LaunchUtils/ScriptCaller.sh' if path is None else path
         self.world = COMPUTERRARIA_DIR + 'computer.wld' if world is None else world
         self.port = port
         self.inplace = inplace
         self.verbose = verbose
+        self.terracc = terracc
+        self.lazy = lazy
 
 
         # World specific config
@@ -95,7 +101,7 @@ def __init__(
             'dummy40-2': (3177, 49),
             'dummy40-3': (3129, 49),
             'dummy-tl': (3089, 53), # Individual dummies, top left
-            'clk': (3201, 158), # manual clock
+            'clk': (3194, 153), # manual clock
             'reset': (3198, 156), # reset
             'zdb': (3243, 226), # zero data bus
             'zmem': (3250, 374), # zero memory select
@@ -132,16 +138,21 @@ def start(self):
 
         if self.verbose:
             self.process.logfile = sys.stdout.buffer
+        else:
+            self.process.logfile = open(TMP_DIR + 'tinterface.log', 'wb')
 
         self.process.expect('Server started')
+        print('Server started')
         time.sleep(0.2)
         self.process.sendline('init')
         time.sleep(0.2)
         self.process.sendline(f'bin config {self.config_x.to_str()} {self.config_y.to_str()}')
         time.sleep(0.2)
         self.clock_start(self.triggers['clk'], -1)
         time.sleep(0.2)
-
+        if self.terracc:
+            self.compile(lazy=self.lazy)
+            print('terracc compiled')
 
     def stop(self):
         """Stops the server and cleans up"""
@@ -164,13 +175,18 @@ def config(self, config_x, config_y):
         assert(self.running())
         self.process.sendline(f'bin config {self.config_x.to_str()} {self.config_y.to_str()}')
 
+    def compile(self, lazy=False):
+        self.process.sendline('accel compile lazy' if lazy else 'accel compile')
+        self.process.expect('terracc enabled', timeout=120)
+
     def write_bin(self, file: str):
         """Writes given file to the world, if given an elf it will convert to bin in place"""
         assert(self.running())
         f, ext = os.path.splitext(file)
         assert(ext == '.bin' or ext == '.elf' or ext == '.txt')
         binfile = f + '.bin'
         txtfile = f + '.txt'
+
         if ext == '.elf':
             objcopy = ''
             if shutil.which('rust-objcopy'):
@@ -187,6 +203,7 @@ def write_bin(self, file: str):
                 # hexdump -ve '1/1 "%.2x "' | head -c -1 > 
                 # Need to trim since WireHead doesn't like a trailing space
                 f.write(pexpect.run(f'hexdump -ve \'1/1 "%.2x "\' {binfile}').decode('utf-8')[:-1])
+
         # Sync here to avoid accidentally writing without syncing first
         self.sync()
         # WireHead has weird case sensitive glitch that I don't feel like fixing
@@ -202,6 +219,7 @@ def read_bin(self, file: str, force=False):
         if not force:
             # Unless forced make sure you don't overwrite a non txt file
             assert(ext == '.txt')
+
         # Sync here to avoid accidentally reading without syncing first
         self.sync()
         # WireHead has weird case sensitive glitch that I don't feel like fixing
@@ -293,10 +311,12 @@ def reset_state(self):
         """Resets all state of the computer to a blank slate except ram"""
         assert(self.running())
 
+        self.sync()
         tries = 0
         # Prevent trying to reset state while in the middle of execution
         while not self.read(self.tiles['inexec']):
             self.trigger(self.triggers['clk'])
+            self.sync()
             tries += 1
             # Most clock cycles of an instructions is 3
             if tries >= 3: 
@@ -310,6 +330,7 @@ def reset_state(self):
         self.trigger(self.triggers['zdb'])
         self.trigger(self.triggers['zmem'])
         self.trigger(self.triggers['lpc'])
+        self.sync()
 
     def write_zeros(self):
         """Writes all zeros to memory"""
@@ -342,57 +363,52 @@ def set_freq(self, f: int):
                                  self.triggers['dummy-tl'][1] + y * self.dummies_gap[1]))
                     n_cur -= 1
 
-    def run(self, prog_file: str, out_file: str, clock_cycles=50000):
+    def run(self, prog_file: str, out_file: str, clock_cycles=50000, timeout_s=300):
         """
         Runs prog_file and returns output to out_file
-
-        This method attempts to be clever about the frequency given to the cpu to minimize lag
+        Assumes world is in ready state
         """
         assert(self.running())
-        self.reset_state()
         self.write_bin(prog_file)
 
+        time.sleep(0.5)
+
+        if self.terracc:
+            print('Started compiling')
+            self.compile(lazy=False)
+            print('Finished compiling')
+
         first_cc = self.clock_count()
 
         # Number of dummies active
         n_cur = 120
         self.set_freq(n_cur * FPS)
 
-        # Delay between dynamic adjustment of clock speed
-        # Doesn't matter too much as long as triggers aren't super frequent
-        delay = 3
-
-        # Clock cycle tracker
-        cc = self.clock_count()
-        last_cc = cc
-        while cc - first_cc < clock_cycles:
-            # Do actual computation
-            time.sleep(delay)
-            last_cc = cc
-            cc = self.clock_count()
-
-            # Calculate whether we're lagging or not
-            expected_count = FPS * delay * n_cur
-            ratio = (cc - last_cc) / expected_count
-            # ~0.9 fps is a reasonable amount of lag
-            if ratio < 0.90:
-                n_next = int(n_cur * ratio / 0.9)
-                n_cur = max(1, n_next)
-            else:
-                n_cur = min(self.num_dummies[0] * self.num_dummies[1], n_cur + 5)
+        self.sync()
 
-            # Adjust frequency
-            print(f'Adjusting frequency to f={n_cur * FPS} Hz (Expected: {expected_count}, cc: {cc}, last_cc: {last_cc}, ratio: {ratio})')
-            self.set_freq(n_cur * FPS)
+        t_start = time.time()
+        while self.clock_count() - first_cc < clock_cycles:
+            time.sleep(1)
+            if self.clock_count() == first_cc:
+                print('Clock not progressing! Cancelling run')
+                break
+            if time.time() - t_start > timeout_s:
+                print('Timed out! Cancelling run')
+                break
 
-            cc = self.clock_count()
+        print('Finished execution')
 
         self.set_freq(0)
 
-        self.sync()
         self.reset_state()
+        time.sleep(0.2)
         # read_bin handles syncing
         self.read_bin(out_file)
 
+    def stress_test(self):
+        print('Starting stress test')
+        for _ in tqdm(range(5000)):
+            self.trigger(self.triggers['clk'])
+        print('Finished stress test')