Merge pull request #516 from MikaylaFischler/devel

2024.07.05 Release

.github/workflows/check.yml

@@ -26,4 +26,4 @@ jobs:
           # --no-max-line-length = Disable warnings for long line lengths
           # --exclude-files ...  = Exclude lockbox library (external) and config files
           # --globals ...        = Override all globals overridden in .vscode/settings.json AND 'os' since CraftOS 'os' differs from Lua's 'os'
-          args: . --no-max-line-length -i 121 512 542 --exclude-files ./lockbox/* ./*/config.lua --globals os _HOST bit colors fs http keys parallel periphemu peripheral read rs settings shell term textutils window
+          args: . --no-max-line-length -i 121 512 542 --exclude-files ./lockbox/* --globals os _HOST bit colors fs http keys parallel periphemu peripheral read rs settings shell term textutils window

.github/workflows/manifest.yml

@@ -61,7 +61,7 @@ jobs:
       - name: Generate manifest for devel
         id: manifest-devel
         if: ${{ (success() || failure()) && steps.checkout-devel.outcome == 'success' }}
-        run: python imgen.py
+        run: python build/imgen.py
       - name: Save devel's manifest
         if: ${{ (success() || failure()) && steps.manifest-devel.outcome == 'success' }}
         run: mv install_manifest.json deploy/manifests/devel

.gitignore

@@ -1,2 +1,2 @@
-_notes/
+_*/
 /*program.sh

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright © 2022 - 2024 Mikayla Fischler
+Copyright 2022 - 2024 Mikayla Fischler
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

build/_offline.lua

@@ -0,0 +1,118 @@
+---@diagnostic disable: undefined-global
+-- luacheck: push ignore install_manifest ccmsi_offline app_files dep_files lgray green white
+
+local b64_lookup = {
+    ['A'] = 0,  ['B'] = 1,  ['C'] = 2,  ['D'] = 3,  ['E'] = 4,  ['F'] = 5,  ['G'] = 6,  ['H'] = 7,  ['I'] = 8,  ['J'] = 9,  ['K'] = 10, ['L'] = 11, ['M'] = 12, ['N'] = 13, ['O'] = 14, ['P'] = 15, ['Q'] = 16, ['R'] = 17, ['S'] = 18, ['T'] = 19, ['U'] = 20, ['V'] = 21, ['W'] = 22, ['X'] = 23, ['Y'] = 24, ['Z'] = 25,
+    ['a'] = 26, ['b'] = 27, ['c'] = 28, ['d'] = 29, ['e'] = 30, ['f'] = 31, ['g'] = 32, ['h'] = 33, ['i'] = 34, ['j'] = 35, ['k'] = 36, ['l'] = 37, ['m'] = 38, ['n'] = 39, ['o'] = 40, ['p'] = 41, ['q'] = 42, ['r'] = 43, ['s'] = 44, ['t'] = 45, ['u'] = 46, ['v'] = 47, ['w'] = 48, ['x'] = 49, ['y'] = 50, ['z'] = 51,
+    ['0'] = 52, ['1'] = 53, ['2'] = 54, ['3'] = 55, ['4'] = 56, ['5'] = 57, ['6'] = 58, ['7'] = 59, ['8'] = 60, ['9'] = 61, ['+'] = 62, ['/'] = 63
+}
+
+local BYTE  = 0xFF
+local CHAR  = string.char
+local BOR   = bit.bor           ---@type function
+local BAND  = bit.band          ---@type function
+local LSHFT = bit.blshift       ---@type function
+local RSHFT = bit.blogic_rshift ---@type function
+
+-- decode a base64 string
+---@param input string
+local function b64_decode(input)
+---@diagnostic disable-next-line: undefined-field
+    local t_start = os.epoch("local")
+
+    local decoded = {}
+
+    local c_idx, idx = 1, 1
+
+    for _ = 1, input:len() / 4 do
+        local block = input:sub(idx, idx + 4)
+        local word = 0x0
+
+        -- build the 24-bit sequence from the 4 characters
+        for i = 1, 4 do
+            local num = b64_lookup[block:sub(i, i)]
+
+            if num then
+                word = BOR(word, LSHFT(b64_lookup[block:sub(i, i)], (4 - i) * 6))
+            end
+        end
+
+        -- decode the 24-bit sequence as 8 bytes
+        for i = 1, 3 do
+            local char = BAND(BYTE, RSHFT(word, (3 - i) * 8))
+
+            if char ~= 0 then
+                decoded[c_idx] = CHAR(char)
+                c_idx = c_idx + 1
+            end
+        end
+
+        idx = idx + 4
+    end
+
+---@diagnostic disable-next-line: undefined-field
+    local elapsed = (os.epoch("local") - t_start)
+    local decoded_str = table.concat(decoded)
+
+    return decoded_str, elapsed
+end
+
+-- write files recursively from base64 encodings in a table
+---@param files table
+---@param path string
+local function write_files(files, path)
+    fs.makeDir(path)
+
+    for k, v in pairs(files) do
+        if type(v) == "table" then
+            if k == "system" then
+                -- write system files to root
+                write_files(v, "/")
+            else
+                -- descend into directories
+                write_files(v, path .. "/" .. k .. "/")
+            end
+
+---@diagnostic disable-next-line: undefined-field
+            os.sleep(0.05)
+        else
+            local handle = fs.open(path .. k, "w")
+            local text, time = b64_decode(v)
+
+            print("decoded '" .. k .. "' in " .. time .. "ms")
+
+            handle.write(text)
+            handle.close()
+        end
+    end
+end
+
+-- write installation manifiest and offline install manager
+local function write_install()
+    local handle = fs.open("install_manifest.json", "w")
+    handle.write(b64_decode(install_manifest))
+    handle.close()
+
+    handle = fs.open("ccmsim.lua", "w")
+    handle.write(b64_decode(ccmsi_offline))
+    handle.close()
+end
+
+lgray()
+
+-- write both app and dependency files
+write_files(app_files, "/")
+write_files(dep_files, "/")
+
+-- write an install manifest and the offline installer
+write_install()
+
+green()
+print("Done!")
+white()
+print("All files have been installed. The app can be started with 'startup' and configured with 'configure'.")
+lgray()
+print("Hint: You can use 'ccmsim' to manage your off-line installation.")
+white()
+
+--luacheck: pop

build/bundle.py

@@ -0,0 +1,214 @@
+import base64
+import json
+import os
+import subprocess
+import sys
+
+path_prefix = "./_minified/"
+
+# get git build info
+build = subprocess.check_output(["git", "describe", "--tags"]).strip().decode('UTF-8')
+
+# list files in a directory
+def list_files(path):
+    list = []
+
+    for (root, dirs, files) in os.walk(path):
+        for f in files:
+            list.append((root[2:] + "/" + f).replace('\\','/'))
+
+    return list
+
+# recursively encode files with base64
+def encode_recursive(path):
+    list = {}
+
+    for item in os.listdir(path):
+        item_path = path + '/' + item
+
+        if os.path.isfile(item_path):
+            handle = open(item_path, 'r')
+            list[item] = base64.b64encode(bytes(handle.read(), 'UTF-8')).decode('ASCII')
+            handle.close()
+        else:
+            list[item] = encode_recursive(item_path)
+
+    return list
+
+# encode listed files with base64
+def encode_files(files):
+    list = {}
+
+    for item in files:
+        item_path = path_prefix + './' + item
+
+        handle = open(item_path, 'r')
+        list[item] = base64.b64encode(bytes(handle.read(), 'UTF-8')).decode('ASCII')
+        handle.close()
+
+    return list
+
+# get the version of an application at the provided path
+def get_version(path, is_lib = False):
+    ver = ""
+    string = ".version = \""
+
+    if not is_lib:
+        string = "_VERSION = \""
+
+    f = open(path, "r")
+
+    for line in f:
+        pos = line.find(string)
+        if pos >= 0:
+            ver = line[(pos + len(string)):(len(line) - 2)]
+            break
+
+    f.close()
+
+    return ver
+
+# file manifest (reflects imgen.py)
+manifest = {
+    "common_versions" : {
+        "bootloader" : get_version("./startup.lua"),
+        "common" : get_version("./scada-common/util.lua", True),
+        "comms" : get_version("./scada-common/comms.lua", True),
+        "graphics" : get_version("./graphics/core.lua", True),
+        "lockbox" : get_version("./lockbox/init.lua", True),
+    },
+    "app_versions" : {
+        "reactor-plc" : get_version("./reactor-plc/startup.lua"),
+        "rtu" : get_version("./rtu/startup.lua"),
+        "supervisor" : get_version("./supervisor/startup.lua"),
+        "coordinator" : get_version("./coordinator/startup.lua"),
+        "pocket" : get_version("./pocket/startup.lua")
+    },
+    "files" : {
+        # common files
+        "system" : encode_files([ "initenv.lua", "startup.lua", "configure.lua", "LICENSE" ]),
+        "scada-common" : encode_recursive(path_prefix + "./scada-common"),
+        "graphics" : encode_recursive(path_prefix + "./graphics"),
+        "lockbox" : encode_recursive(path_prefix + "./lockbox"),
+        # platform files
+        "reactor-plc" : encode_recursive(path_prefix + "./reactor-plc"),
+        "rtu" : encode_recursive(path_prefix + "./rtu"),
+        "supervisor" : encode_recursive(path_prefix + "./supervisor"),
+        "coordinator" : encode_recursive(path_prefix + "./coordinator"),
+        "pocket" : encode_recursive(path_prefix + "./pocket"),
+    },
+    "install_files" : {
+        # common files
+        "system" : [ "initenv.lua", "startup.lua", "configure.lua", "LICENSE" ],
+        "scada-common" : list_files("./scada-common"),
+        "graphics" : list_files("./graphics"),
+        "lockbox" : list_files("./lockbox"),
+        # platform files
+        "reactor-plc" : list_files("./reactor-plc"),
+        "rtu" : list_files("./rtu"),
+        "supervisor" : list_files("./supervisor"),
+        "coordinator" : list_files("./coordinator"),
+        "pocket" : list_files("./pocket"),
+    },
+    "depends" : [ "system", "scada-common", "graphics", "lockbox" ]
+}
+
+# write the application installation items as Lua tables
+def write_items(body, items, indent):
+    indent_str = " " * indent
+    for key, value in items.items():
+        if isinstance(value, str):
+            body = body + f"{indent_str}['{key}'] = \"{value}\",\n"
+        else:
+            body = body + f"{indent_str}['{key}'] = {{\n"
+            body = write_items(body, value, indent + 4)
+            body = body + f"{indent_str}}},\n"
+
+    return body
+
+# create output directory
+if not os.path.exists("./BUNDLE"):
+    os.makedirs("./BUNDLE")
+
+# get offline installer
+ccmsim_file = open("./build/ccmsim.lua", "r")
+ccmsim_script = ccmsim_file.read()
+ccmsim_file.close()
+
+# create dependency bundled file
+dep_file = "common_" + build + ".lua"
+f_d = open("./BUNDLE/" + dep_file, "w")
+
+body_b = "local dep_files = {\n"
+
+for depend in manifest["depends"]:
+    body_b = body_b + write_items("", { f"{depend}": manifest["files"][depend] }, 4)
+body_b = body_b + "}\n"
+
+body_b = body_b + f"""
+if select("#", ...) == 0 then
+    term.setTextColor(colors.red)
+    print("You must run the other file you should have uploaded (it has the app in its name).")
+    term.setTextColor(colors.white)
+end
+
+return dep_files
+"""
+
+f_d.write(body_b)
+f_d.close()
+
+# application bundled files
+for app in [ "reactor-plc", "rtu", "supervisor", "coordinator", "pocket" ]:
+    app_file = app + "_" + build + ".lua"
+
+    f_script = open("./build/_offline.lua", "r")
+    script = f_script.read()
+    f_script.close()
+
+    f_a = open("./BUNDLE/" + app_file, "w")
+
+    body_a = "local app_files = {\n"
+
+    body_a = body_a + write_items("", { f"{app}": manifest["files"][app] }, 4) + "}\n"
+
+    versions = manifest["common_versions"].copy()
+    versions[app] = manifest["app_versions"][app]
+
+    depends = manifest["depends"].copy()
+    depends.append(app)
+
+    install_manifest = json.dumps({ "versions" : versions, "files" : manifest["install_files"], "depends" : depends })
+
+    body_a = body_a + f"""
+-- install manifest JSON and offline installer
+local install_manifest = "{base64.b64encode(bytes(install_manifest, 'UTF-8')).decode('ASCII')}"
+local ccmsi_offline = "{base64.b64encode(bytes(ccmsim_script, 'UTF-8')).decode('ASCII')}"
+
+local function red() term.setTextColor(colors.red) end
+local function green() term.setTextColor(colors.green) end
+local function white() term.setTextColor(colors.white) end
+local function lgray() term.setTextColor(colors.lightGray) end
+
+if not fs.exists("{dep_file}") then
+    red()
+    print("Missing '{dep_file}'! Please upload it, then run this file again.")
+    white()
+    return
+end
+
+-- rename the dependency file
+fs.move("{dep_file}", "install_depends.lua")
+
+-- load the other file
+local dep_files = require("install_depends")
+
+-- delete the uploaded files to free up space to actually install
+fs.delete("{app_file}")
+fs.delete("install_depends.lua")
+
+-- get started installing
+{script}"""
+
+    f_a.write(body_a)
+    f_a.close()