Merge pull request #78 from FlyingBananaTree/patch-1

Add 16.0.0 pkg1

Makefile

@@ -9,9 +9,9 @@ include $(DEVKITARM)/base_rules
 ################################################################################
 
 IPL_LOAD_ADDR := 0x40008000
-LPVERSION_MAJOR := 3
+LPVERSION_MAJOR := 4
 LPVERSION_MINOR := 0
-LPVERSION_BUGFX := 6
+LPVERSION_BUGFX := 1
 LPVERSION := \"$(LPVERSION_MAJOR).$(LPVERSION_MINOR).$(LPVERSION_BUGFX)\"
 
 ################################################################################
@@ -113,9 +113,9 @@ $(BUILDDIR)/$(TARGET)/script/builtin.c: scripts/*.te
 	@mkdir -p "$(@D)"
 	@mkdir -p "$(BUILDDIR)/$(TARGET)/scripts"
 ifeq ($(OS),Windows_NT)
-	@py ts-minifier.py -d "$(BUILDDIR)/$(TARGET)/scripts" $(wildcard scripts/*.te)
+	@py ts-minifier.py --such-meme -d "$(BUILDDIR)/$(TARGET)/scripts" $(wildcard scripts/*.te)
 	@py te2c.py "$(BUILDDIR)/$(TARGET)/script/builtin" "$(BUILDDIR)/$(TARGET)/scripts"
 else
-	@python3 ts-minifier.py -d "$(BUILDDIR)/$(TARGET)/scripts" $(wildcard scripts/*.te)
+	@python3 ts-minifier.py --such-meme -d "$(BUILDDIR)/$(TARGET)/scripts" $(wildcard scripts/*.te)
 	@python3 te2c.py "$(BUILDDIR)/$(TARGET)/script/builtin" "$(BUILDDIR)/$(TARGET)/scripts"
 endif

loader/Makefile

@@ -27,8 +27,11 @@ OBJS = $(addprefix $(BUILDDIR)/$(TARGET)/, \
 CUSTOMDEFINES := -DBL_MAGIC=$(IPL_MAGIC)
 CUSTOMDEFINES += -DBL_VER_MJ=$(BLVERSION_MAJOR) -DBL_VER_MN=$(BLVERSION_MINOR) -DBL_VER_HF=$(BLVERSION_HOTFX) -DBL_RESERVED=$(BLVERSION_RSVD)
 
+#TODO: Considering reinstating some of these when pointer warnings have been fixed.
+WARNINGS := -Wall -Wno-array-bounds -Wno-stringop-overflow
+
 ARCH := -march=armv4t -mtune=arm7tdmi -mthumb-interwork
-CFLAGS = $(ARCH) -O2 -g -nostdlib -ffunction-sections -fdata-sections -fomit-frame-pointer -std=gnu11 -Wall $(CUSTOMDEFINES)
+CFLAGS = $(ARCH) -O2 -g -nostdlib -ffunction-sections -fdata-sections -fomit-frame-pointer -std=gnu11 $(WARNINGS) $(CUSTOMDEFINES)
 LDFLAGS = $(ARCH) -nostartfiles -lgcc -Wl,--nmagic,--gc-sections -Xlinker --defsym=LDR_LOAD_ADDR=$(LDR_LOAD_ADDR)
 
 ################################################################################

loader/link.ld

@@ -5,7 +5,7 @@ SECTIONS {
 	. = __ipl_start;
 	.text : {
 		*(.text._start);
-		*(._boot_cfg);
+		KEEP(*(._boot_cfg));
 		*(.text*);
 	}
 	.data : {

loader/loader.c

@@ -33,9 +33,6 @@ boot_cfg_t __attribute__((section ("._boot_cfg"))) b_cfg;
 
 void loader_main()
 {
-	// Preserve sections.
-	__asm__ ("" : : "" (b_cfg));
-
 	// Preliminary BPMP clocks init.
 	CLOCK(CLK_RST_CONTROLLER_CLK_SYSTEM_RATE) = 0x10;          // Set HCLK div to 2 and PCLK div to 1.
 	CLOCK(CLK_RST_CONTROLLER_CLK_SOURCE_SYS) = 0;              // Set SCLK div to 1.
@@ -45,7 +42,8 @@ void loader_main()
 	CLOCK(CLK_RST_CONTROLLER_SCLK_BURST_POLICY) = 0x20003333;  // Set SCLK to PLLP_OUT (408MHz).
 
 	// Get Loader and Payload size.
-	u32 payload_size = sizeof(payload_00) + sizeof(payload_01);
+	u32 payload_size = sizeof(payload_00) + sizeof(payload_01);             // Actual payload size.
+	payload_size += (u32)payload_01 - (u32)payload_00 - sizeof(payload_00); // Add array alignment.
 	u32 *payload_addr = (u32 *)payload_00;
 
 	// Relocate payload to a safer place.
@@ -60,16 +58,20 @@ void loader_main()
 		bytes--;
 	}
 
-	// Uncompress payload parts.
+	// Set source address of the first part.
 	u8 *src_addr = (void *)(IPL_RELOC_TOP - ALIGN(payload_size, 4));
-	u32 pos = LZ_Uncompress((const u8 *)src_addr, (u8*)IPL_LOAD_ADDR, sizeof(payload_00));
-	src_addr += (u32)payload_01 - (u32)payload_00;
-	LZ_Uncompress((const u8 *)src_addr, (u8*)IPL_LOAD_ADDR + pos, sizeof(payload_01));
+	// Uncompress first part.
+	u32 dst_pos = LZ_Uncompress((const u8 *)src_addr, (u8*)IPL_LOAD_ADDR, sizeof(payload_00));
 
-	// Copy over boot configuration storage in case it was set.
+	// Set source address of the second part. Includes array alignment.
+	src_addr += (u32)payload_01 - (u32)payload_00;
+	// Uncompress second part.
+	LZ_Uncompress((const u8 *)src_addr, (u8*)IPL_LOAD_ADDR + dst_pos, sizeof(payload_01));
+
+	// Copy over boot configuration storage.
 	memcpy((u8 *)(IPL_LOAD_ADDR + IPL_PATCHED_RELOC_SZ), &b_cfg, sizeof(boot_cfg_t));
 
-	// Chainload.
+	// Chainload into uncompressed payload.
 	void (*ipl_ptr)() = (void *)IPL_LOAD_ADDR;
 	(*ipl_ptr)();
 

scripts/SystemWipe.te

@@ -1,3 +1,4 @@
+#REQUIRE KEYS
 is=["8000000000000120","80000000000000d1","8000000000000047"]
 p=println
 pr=print

source/fs/menus/filemenu.c

@@ -90,7 +90,8 @@ void RunScript(char *path, FSEntry_t entry){
     char *thing = CombinePaths(path, entry.name);
     u32 size;
     char *script = sd_file_read(thing, &size);
-    TConf.scriptCWD = thing;
+    free(thing);
+    TConf.scriptCWD = path;
 
     if (!script)
         return;
@@ -110,7 +111,6 @@ void RunScript(char *path, FSEntry_t entry){
     exitFunction(ret.main.operations.data, ret.main.operations.count);
     vecFree(ret.staticVarHolder);
     vecFree(ret.main.operations);
-    free(thing);
 }
 
 void RenameFile(char *path, FSEntry_t entry){

source/hos/pkg1.c

@@ -42,6 +42,11 @@ static const pkg1_id_t _pkg1_ids[] = {
 	{ "20210129", 10}, //12.0.0 - 12.0.1
 	{ "20210422", 10}, //12.0.2 - 12.0.3
 	{ "20210607", 11}, //12.1.0
+	{ "20210805", 12}, //13.0.0 - 13.2.0
+	{ "20220105", 12}, //13.2.1
+	{ "20220209", 13}, //14.0.0 - 14.1.2
+	{ "20220801", 14}, //15.0.0 - 15.0.1
+	{ "20230111", 15}, //16.0.0+
 	{ NULL } //End.
 };
 

source/keys/keys.c

@@ -240,9 +240,9 @@ static ALWAYS_INLINE u8 *_read_pkg1(const pkg1_id_t **pkg1_id) {
     *pkg1_id = pkg1_identify(pkg1 + pk1_offset);
     if (!*pkg1_id) {
         DPRINTF("Unknown pkg1 version.\n Make sure you have the latest Lockpick_RCM.\n If a new firmware version just came out,\n Lockpick_RCM must be updated.\n Check Github for new release.");
-        //gfx_hexdump(0, pkg1, 0x20);
+        //gfx_hexdump(0, pkg1 + pk1_offset, 0x20);
         char pkg1txt[16] = {0};
-        memcpy(pkg1txt, pkg1 + pk1_offset +  0x10, 15);
+        memcpy(pkg1txt, pkg1 + pk1_offset + 0x10, 14);
         gfx_printf("Unknown pkg1 version\nMake sure you have the latest version of TegraExplorer\n\nPKG1: '%s'\n", pkg1txt);
         return NULL;
     }

source/script/arrayClass.c

@@ -180,11 +180,11 @@ ClassFunction(arrayContains) {
 	
 		if (caller->variableType == StringArrayClass) {
 			if (!strcmp(arg->string.value, iter.string.value))
-				return newIntVariablePtr(1);
+				return newIntVariablePtr(i + 1);
 		}
 		else {
 			if (arg->integer.value == iter.integer.value)
-				return newIntVariablePtr(1);
+				return newIntVariablePtr(i + 1);
 		}
 	}
 

source/script/standardLibrary.c

@@ -234,13 +234,8 @@ char *powNames[] = {
 	"volminus",
 };
 
-// Takes [int]. Returns dict[a,b,x,y,down,up,right,left,power,volplus,volminus,raw]. int: mask for hidWaitMask 
-ClassFunction(stdPauseMask){
+Variable_t *hidToVar(u32 raw){
 	Variable_t ret = {.variableType = DictionaryClass, .dictionary.vector = newVec(sizeof(Dict_t), 9)};
-	Input_t *i = hidWaitMask((u32)getIntValue(*args));
-	
-	u32 raw = i->buttons;
-
 	addIntToDict(&ret, "raw", raw);
 
 	for (int i = 0; i < ARRAY_SIZE(abxyNames); i++){
@@ -265,6 +260,19 @@ ClassFunction(stdPauseMask){
 	return copyVariableToPtr(ret);
 }
 
+ClassFunction(stdRead){
+	Input_t *i = hidRead();
+	u32 raw = i->buttons;
+	return hidToVar(raw); 
+}
+
+// Takes [int]. Returns dict[a,b,x,y,down,up,right,left,power,volplus,volminus,raw]. int: mask for hidWaitMask 
+ClassFunction(stdPauseMask){
+	Input_t *i = hidWaitMask((u32)getIntValue(*args));
+	u32 raw = i->buttons;
+	return hidToVar(raw);
+}
+
 // Takes none. Returns dict (same as stdPauseMask). 
 ClassFunction(stdPause){
 	Variable_t a = {.integer.value = 0xFFFFFFFF};
@@ -482,6 +490,7 @@ STUBBED(stdMkdir)
 STUBBED(stdGetMemUsage)
 STUBBED(stdGetNcaType)
 STUBBED(stdPause)
+STUBBED(stdRead)
 STUBBED(stdPauseMask)
 STUBBED(stdColor)
 STUBBED(stdMenuFull)
@@ -541,6 +550,7 @@ ClassFunctionTableEntry_t standardFunctionDefenitions[] = {
 	{"timer", stdGetMs, 0, 0},
 	{"pause", stdPauseMask, 1, threeIntsStd},
 	{"pause", stdPause, 0, 0},
+	{"hidread", stdRead, 0, 0},
 	{"color", stdColor, 1, threeIntsStd},
 	{"menu", stdMenuFull, 3, menuArgsStd},
 	{"menu", stdMenuFull, 2, menuArgsStd},

source/script/unsolvedArrayClass.c

@@ -152,7 +152,12 @@ Variable_t createUnsolvedArrayVariable(Function_t* f) {
 	return var;
 }
 
+ClassFunction(retZero){
+	return newIntVariablePtr(0);
+}
+
 u8 anotherOneVarArg[] = { VARARGCOUNT };
+u8 unsolvedArrayStr[] = { StringClass };
 
 ClassFunction(createTypedArray) {
 	Vector_t v = { 0 };
@@ -180,6 +185,7 @@ ClassFunction(createTypedArray) {
 ClassFunctionTableEntry_t unsolvedArrayFunctions[] = {
 	{"+", createTypedArray, 1, anotherOneVarArg},
 	{"add", createTypedArray, 1, anotherOneVarArg},
+	{"len", retZero, 0, 0},
 };
 
 Variable_t getUnsolvedArrayMember(Variable_t* var, char* memberName) {

ts-minifier.py

@@ -1,29 +1,55 @@
 # Copyright (c) 2021 bleck9999
 # https://github.com/bleck9999/ts-minifier
-# Version: 91345ce1
-
+# Version: 49befe92
 import argparse
-import re
+import itertools
+import logging
+from os import path
+from string import ascii_letters, digits, hexdigits
 
-# if is not included because it's already 2 characters
-sub_funcs = {'while': "_h", 'print': "_p", 'println': "_l", 'mountsys': "_s", 'mountemu': "_e", 'readsave': "_r",
-             'exit': "_q", 'break': "_b", 'dict': "_d", 'setpixel': "_y", 'readdir': "_i", 'copyfile': "_c",
-             'mkdir': "_k", 'ncatype': "_n", 'pause': "_w", 'color': "_a", 'menu': "__", 'emu': "_u",
-             'clear': "_x", 'timer': "_t", 'deldir': "_g", 'fsexists': "_f", 'delfile': "_z", "copydir": "c_",
-             "movefile": "_v", "payload": "_j", "readfile": "_o", "writefile": "_W", "setpixels": "_Y", "printpos": "_P",
-             "emmcread": "_E", "emmcwrite": "_F", "emummcread": "_R", "emummcwrite": "_S", "escapepath": "_X",
-             "combinepath": "_A", "cwd": "_D", "power": "_O", "fuse_patched": "_M", "fuse_hwtype": "_N"}
-replace_functions = False
+auto_replace = False
+verbose = False
+stdlib = ['if', 'while', 'print', 'println', 'mountsys', 'mountemu', 'readsave', 'exit', 'break', 'dict', 'setpixel',
+          'readdir', 'copyfile', 'mkdir', 'ncatype', 'pause', 'color', 'menu', 'emu', 'clear', 'timer', 'deldir',
+          'fsexists', 'delfile', 'copydir', 'movefile', 'payload', 'readfile', 'writefile', 'setpixels', 'printpos',
+          'emmcread', 'emmcwrite', 'emummcread', 'emummcwrite', 'escapepath', 'combinepath', 'cwd', 'power',
+          'fuse_patched', 'fuse_hwtype', 'hidread']
 
 
-def wantsumspace(s: str):
-    for c in s.lower():
-        if (ord(c) < 97 or ord(c) > 122) and (ord(c) != 95) and not (c.isnumeric()):
+class Code:
+    def __init__(self, strings, comments, script):
+        counter = 0
+        strings_comments = sorted(strings + comments)
+
+        bound = 0
+        code = []
+        for strcom in strings_comments:
+            code.append((bound, strcom[0], script[bound:strcom[0]]))
+            bound = strcom[1]
+        code.append((bound, len(script), script[bound:]))
+        self.sections = sorted(strings_comments + code)
+        self.strings = strings
+        self.comments = comments
+        self.code = code
+        self.varstrs = []
+        self.rawcode = "".join([x[2] for x in sorted(self.code + self.strings)])
+
+
+def isidentifier(s: str):
+    for c in s:
+        if c not in (ascii_letters + '_' + digits):
             return False
     return True
 
 
-def commentstartswhere(s: str):
+def iswhitespace(s: str):
+    for c in s:
+        if c not in (' ', '\t', '\n'):
+            return False
+    return True
+
+
+def hascomment(s: str):
     quoted = False
     for c in range(len(s)):
         if s[c] == '"':
@@ -33,29 +59,310 @@ def commentstartswhere(s: str):
     return None
 
 
-def minify(script: str):
-    # currently ts does not seem to allow 's to mark a quote
-    # (https://github.com/suchmememanyskill/TegraExplorer/blob/tsv3/source/script/parser.c#L173)
-    # im fine with that, it makes doing this a lot easier
-    # strings = script.split(sep='"')
+def parser(script: str):
+    comments = []  # [(start, end, content)]
+    strings = []
+    commented = False
+    quoted = False
+    strstart = -1
+    commentstart = -1
+    for c in range(len(script)):
+        if script[c] == '#' and not quoted:
+            commented = True
+            commentstart = c
+        elif (script[c] == '\n' and not quoted) and commented:
+            comments.append((commentstart, c + 1, script[commentstart:c + 1]))
+            commented = False
+        elif script[c] == '"' and not commented:
+            if not quoted:
+                strstart = c
+                quoted = True
+            else:
+                strings.append((strstart, c + 1, script[strstart:c + 1]))
+                quoted = False
+
+    script = Code(strings, comments, script)
+
+    # couple notes:
+    # we minify the script a little before parsing, so there is no unnecessary whitespace or comments
+    # we are assuming the input script is valid syntax
+    userobjects = {}
+    usages = {}
+    hexxed = False
+    ismember = False
+    quoted = False
+    strscript = script.rawcode + ' '
+    # the space will get removed after the second pass of whitespacent, but for now it prevents not detecting the
+    # last identifier in a script (eg if script.rawcode was "a=12" the 12 wouldn't be detected without the trailing ' ')
+    start = len(strscript) + 1
+    for ch in range(len(strscript)):
+        if (strscript[ch - 1] == '0' and strscript[ch] == 'x') and not quoted:
+            hexxed = True
+        elif isidentifier(strscript[ch]) and not (hexxed or quoted):
+            if start > ch:
+                start = ch
+            else:
+                pass
+        elif hexxed and strscript[ch] not in hexdigits:
+            hexxed = False
+        elif strscript[ch] == '"':
+            quoted = not quoted
+        elif not quoted:
+            if start != len(strscript) + 1 and not ismember:  # if we actually had an identifier before this char
+                identifier = strscript[start:ch]  # and this isnt a member of anything
+                if identifier in usages:
+                    usages[identifier].append(start)
+                elif identifier.isnumeric():  # numbers are legally valid identifiers because fuckyou
+                    usages[identifier] = [start]
+                    userobjects[identifier] = "INT"
+                elif identifier == "0x":
+                    pass
+                elif strscript[ch] == '=' and strscript[ch + 1] != '=':
+                    isfunc = strscript[ch + 1] == '{'
+                    userobjects[identifier] = "func" if isfunc else "var"
+                    usages[identifier] = [start]  # declaration is a usage because i cant be arsed
+                else:  # not an assignment (or member) but also haven't seen this name before
+                    usages[identifier] = [start]
+                    # fuck it we are using a fucking list of fucking stdlib functions i just fucking cant im adding tsv3
+                    # to the fucking esolangs wiki have a good day
+                    if identifier not in stdlib:
+                        userobjects[identifier] = "var"
+            if strscript[ch] == '.':
+                ismember = True
+            elif strscript[ch] == '(':
+                if ismember:
+                    if "foreach" == strscript[start:ch]:  # array.foreach takes a variable name as an arg (blame meme)
+                        for i, string in enumerate(script.strings):
+                            if string[0] == ch + (script.comments[-1][1] if script.comments else 0) + 1:
+                                script.varstrs.append(string)
+                                script.strings.pop(i)
+                                break
+                    else:
+                        ismember = False
+                        pass
+            elif strscript[ch] in ')}]':
+                ismember = strscript[ch + 1] == '.'
+            start = len(strscript) + 1
+
+    return script, userobjects, usages
+
+
+def minify(script: Code, userobjects, usages):
+    # the space saved by an alias is the amount of characters currently used by calling the function (uses*len(func))
+    # minus the amount of characters it would take to define an alias (len(alias)+len(func)+2), with the 2 being for the
+    # equals and the whitespace needed for a definition
+    # the same principle also applies to introducing a variable for string literals, though since a literal requires
+    # having "s around it then it's uses*(len(str)+2) - (len(minName)+len(str)+4) instead
+    #                                                                          ^ 2 for = and whitespace, 2 for ""
+    #
+    # obviously for a rename you're already defining it so it's just the difference between lengths multiplied by uses
+    short_idents = [x for x in (ascii_letters + '_')] + \
+                   [x[0] + x[1] for x in itertools.product(ascii_letters + '_', repeat=2)]
+    short_idents.pop(short_idents.index("if"))
+    mcode = script.rawcode
+    aliases = []
+    logging.info("Renaming user functions and variables" if auto_replace else
+                 "Checking user function and variable names")
+    for uo in [x for x in userobjects]:
+        if userobjects[uo] not in ["var", "func"]:
+            continue
+        tmpcode = ""
+        otype = userobjects[uo]
+        uses = len(usages[uo])
+        uolen = len(uo)
+        if uolen > 1:
+            candidates = short_idents
+            minName = ''
+            if uolen == 2:
+                candidates = short_idents[:53]
+            for i in candidates:
+                if i not in userobjects:
+                    minName = i
+                    userobjects[minName] = "TRN"
+                    break
+            if not minName:
+                logging.info(f"{'Function' if otype == 'func' else 'Variable'} name {uo} could be shortened but "
+                             f"no available names found (would save {uses} bytes)")
+                continue
+                # we assume that nobody is insane enough to exhaust all *2,808* 2 character names,
+                # instead that uo is len 2 and all the 1 character names are in use (because of that we dont multiply
+                # uses by anything as multiplying by a difference of 1 would be redundant)
+            if not auto_replace:
+                logging.warning(
+                    f"{'Function' if otype == 'func' else 'Variable'} name {uo} could be shortened ({uo}->{minName}, "
+                    f"would save {uses * (uolen - len(minName))} bytes)")
+                continue
+            else:
+                logging.info(f"Renaming {'Function' if otype == 'func' else 'Variable'} {uo} to {minName} "
+                             f"(saving {uses * (uolen - len(minName))} bytes)")
+                diff = uolen - len(minName)
+
+                # the foreach syntax is literally the worst part of ts
+                if otype == "var":
+                    struo = f'"{uo}"'
+                    for varstr in script.varstrs:
+                        if varstr[2] == struo:
+                            logging.info(f"Replacing declaration of {varstr[2]} at {varstr[0]}-{varstr[1]}")
+                            start = varstr[0] - (script.comments[-1][1] if script.comments else 0)
+                            end = varstr[1] - (script.comments[-1][1] if script.comments else 0)
+                            newend = start + len(minName)
+                            mcode = mcode[:newend] + f'{minName}"' + (' ' * diff) + mcode[end:]
+
+                # rather than just blindly str.replace()ing we're going to actually use the character indices that we stored
+                prev = 0
+                for bound in usages[uo]:
+                    tmpcode += mcode[prev:bound] + minName + ' ' * diff
+                    prev = bound + diff + len(minName)
+                # actually shut up about "bound might be referenced before assignment" or show me what possible
+                # execution path that could lead to usages[uo] being an empty list
+                mcode = tmpcode + mcode[bound + diff + len(minName):]
+    logging.info("Aliasing standard library functions" if auto_replace else
+                 "Checking for standard library aliases")
+    for func in usages:
+        tmpcode = ""
+        candidates = short_idents
+        minName = ''
+        savings = 0
+        uses = len(usages[func])
+        if func in userobjects or uses < 2:  # we only want stdlib functions used more than once
+            continue
+        elif func == "if":
+            candidates = short_idents[:53]
+            savings = uses * 2 - 5  # the 5 is how many characters an alias declaration would use (a=if<space>)
+        for i in candidates:
+            if i not in userobjects:
+                minName = i
+                break
+        # once again we assume it's only `if` that could trigger this message
+        # 4 is the minimum amount of uses needed to save space, 1*(uses - 4) is the space it would save
+        if not minName and uses > 4:
+            logging.info(f"Standard library function {func} could be aliased but no available names found "
+                         f"(would save {uses - 4} bytes)")
+        else:
+            if not savings:
+                savings = uses * len(func) - (len(func) + len(minName) + 2)
+            if savings > 0:
+                userobjects[minName] = "TRP"
+                if auto_replace:
+                    logging.info(f"Aliasing standard library function {func} to {minName} (saving {savings} bytes)")
+                    diff = len(func) - len(minName)
+                    prev = 0
+                    for bound in usages[func]:
+                        tmpcode += mcode[prev:bound] + minName + ' ' * diff
+                        prev = bound + diff + len(minName)
+                    mcode = tmpcode + mcode[bound + diff + len(minName):]
+                    aliases.append(f"{minName}={func} ")
+                else:
+                    logging.warning(f"Not aliasing standard library function {func} (would save {savings} bytes)")
+            else:
+                logging.info(f"Not aliasing standard library function {func} (would save {savings} bytes)")
+
     str_reuse = {}
+    for string in script.strings:
+        if string[2] in str_reuse:
+            str_reuse[string[2]].append(string[0])
+        else:
+            str_reuse[string[2]] = [string[0]]
+    logging.info("Introducing variables for reused literals" if auto_replace else
+                 "Checking for reused literals")
+    for string in str_reuse:
+        if string == '"BYTE[]"':
+            # the type specifier for byte arrays is special because it has to be a literal
+            # if it's a variable then it tries to treat it as a string array then shits the bed if the other items are not strings
+            continue
+        tmpcode = ""
+        candidates = short_idents
+        minName = ""
+        uses = len(str_reuse[string])
+        if uses > 1:
+            if len(string) == 2:
+                candidates = short_idents[:53]
+            for i in candidates:
+                if i not in userobjects:
+                    minName = i
+                    break
+            if not minName:
+                savings = len(string) * uses - (len(string) + 5)  # 5 comes from id="{string}"
+                logging.info(f"Could introduce variable for reused string {string} but no available names found "
+                             f"(would save {savings} bytes)")
+                continue
+            # the quotation marks are included in string
+            savings = uses * len(string) - (len(string) + len(minName) + 2)
+            if savings > 0:
+                userobjects[minName] = "TIV"
+                if auto_replace:
+                    # "duplicated code fragment" do i look like i give a shit
+                    logging.info(f"Introducing variable {minName} with value {string} (saving {savings} bytes)")
+                    diff = len(string) - len(minName)
+                    prev = 0
+                    for bound in str_reuse[string]:
+                        bound -= script.comments[-1][1] if script.comments else 0
+                        tmpcode += mcode[prev:bound] + minName + ' ' * diff
+                        prev = bound + diff + len(minName)
+                    mcode = tmpcode + mcode[bound + diff + len(minName):]
+                    aliases.append(f"{minName}={string}")
+                else:
+                    logging.warning(f"Not introducing variable for string {string} reused {uses} times "
+                                    f"(would save {savings} bytes)")
+            else:
+                logging.info(f"Not introducing variable for string {string} reused {uses} times "
+                             f"(would save {savings} bytes)")
+        else:
+            logging.info(f"Not introducing variable for string {string} (only used once)")
+
+    for uint in [x for x in userobjects]:
+        if userobjects[uint] != "INT" or len(uint) < 2:
+            continue
+        candidates = short_idents
+        uses = len(usages[uint])
+        uilen = len(uint)
+        minName = ""
+        tmpcode = ""
+        if uses > 1:
+            if uilen == 2:
+                candidates = short_idents[:53]
+            for i in candidates:
+                if i not in userobjects:
+                    minName = i
+                    break
+            if not minName:
+                # yet another case of "nobody could possibly use up all the 2 char names we hope"
+                savings = uilen * uses - (uilen + 4)  # 4 comes from id={uint}<whitespace>
+                logging.info(f"Could introduce variable for reused integer {uint} but no available names found "
+                             f"(would save {savings} bytes)")
+                continue
+            savings = uilen * uses - (uilen + len(minName) + 2)
+            if savings > 0:
+                userobjects[minName] = "TIV"
+                if auto_replace:
+                    logging.info(f"Introducing variable {minName} with value {uint} (saving {savings} bytes)")
+                    diff = len(uint) - len(minName)
+                    prev = 0
+                    for bound in usages[uint]:
+                        tmpcode += mcode[prev:bound] + minName + ' ' * diff
+                        prev = bound + diff + len(minName)
+                    mcode = tmpcode + mcode[bound + diff + len(minName):]
+                    aliases.append(f"{minName}={uint} ")
+                else:
+                    logging.warning(f"Not introducing variable for integer {uint} reused {uses} times "
+                                    f"(would save {savings} bytes)")
+            else:
+                logging.info(f"Not introducing variable for integer {uint} reused {uses} times "
+                             f"(would save {savings} bytes)")
+        else:
+            logging.info(f"Not introducing variable for integer {uint} (only used once)")
+
+    logging.info("Reintroducing REQUIREs")
+    mcode = "".join([x[2] for x in script.comments]) + "".join(aliases) + mcode
+    return mcode
+
+
+def whitespacent(script: str):
+    # also removes unneeded comments and pushes REQUIREs to the top of the file
     requires = ""
     mcode = ""
-    stl_counts = {}.fromkeys(sub_funcs, 0)
-    # while part < len(strings):
     for line in script.split(sep='\n'):
-        # maybe in future it'll shrink user defined names
-        # dont hold out hope for that because `a.files.foreach("b") {println(b)}` is valid syntax
-        # and i dont have the skill or patience to deal with that
-
-        # # in theory all the even numbered indexes should be outside quotes, so we ignore any parts with an odd index
-        # if part % 2 == 1:
-        #     if strings[part] not in str_reuse:
-        #         str_reuse[strings[part]] = 0
-        #     else:
-        #         str_reuse[strings[part]] += 1
-        #     mcode += f'"{strings[part]}"'
-        start = commentstartswhere(line)
+        start = hascomment(line)
         if start is None:
             start = -1
 
@@ -68,51 +375,39 @@ def minify(script: str):
         # the last character of the line which would lead to several issues
         # however this is desirable when there *is* a comment, since it being exclusive means there isn't a trailing #
         # and if you're wondering about the above check that uses line[start:] this doesn't matter,
-        # one character cant contain an 8 character substring
+        # one character cant contain an 8 character substring so it's not like it'll ever false positive
         if start != -1:
             line = line[:start]
         line = line.split(sep='"')
 
         if len(line) % 2 == 0:
-            print("You appear to have string literals spanning multiple lines. Please seek professional help")
-            raise Exception("Too much hatred")
+            raise Exception("Unmatched quote or hard newline in string")
         part = 0
         while part < len(line):
             # all the odd numbered indexes should be inside quotes
             if part % 2 == 0:
-                if not line[part]:
-                    break
-                for s in sub_funcs:
-                    stl_counts[s] += len(re.findall("(?<!\\.)%s\\(" % s, line[part]))
-                mcode += line[part].replace('\t', '') + ' '
+                if line[part] and not iswhitespace(line[part]):
+                    # turn lots of whitespace into one whitespace with one easy trick!
+                    mcode += ' '.join(line[part].split()) + ' '
             else:
-                if line[part] not in str_reuse:
-                    str_reuse[line[part]] = 0
-                else:
-                    str_reuse[line[part]] += 1
                 mcode += f'"{line[part]}"'
 
             part += 1
 
-
     # tsv3 is still an absolute nightmare
-    # so spaces have a couple edge cases
-    # 1. the - operator which requires space between the right operand
-    # yeah that's right only the right one
-    # thanks meme
-    # 2. between 2 letters
+    # so spaces should be preserved under two situations
+    # 1. the subtraction operator which requires space between the right operand but only if the right operand is a literal
+    # 2. between 2 characters that are valid identifiers (aA-zZ, _ or integers)
     inquote = False
     mmcode = ""
     index = 0
     newline = list(mcode)
     while index < (len(mcode) - 3):
         sec = mcode[index:index + 3]
-        if not inquote and sec[1] == '"':
-            inquote = True
-        elif inquote and sec[1] == '"':
-            inquote = False
+        if sec[1] == '"':
+            inquote = not inquote
         if (sec[1] == ' ') and not inquote:
-            if wantsumspace(sec[0]) and wantsumspace(sec[2]):
+            if (isidentifier(sec[0]) or sec[0].isnumeric()) and (isidentifier(sec[2]) or sec[2].isnumeric()):
                 pass
             elif sec[0] == '-' and sec[2].isnumeric():
                 pass
@@ -121,65 +416,48 @@ def minify(script: str):
         index += 1
     mmcode += ''.join(newline).strip()
 
-    for func in sub_funcs:
-        # space saved here is given by n * (len(func) - len(min_func)) - (len(min_func)+1 + len(func))
-        # as such with one usage space is always lost (len(func)-2 is never > len(func)+3) so dont even try
-        if stl_counts[func] >= 2:
-            savings = stl_counts[func] * (len(func) - 2) - (len(func) + 3)
-            print(f"Replacing all {stl_counts[func]} usages of {func} would save {savings}byte{'s' if savings != 1 else ''}")
-            if (savings < 0) or not replace_functions:
-                print("Savings negative or automatic replacement disabled, continuing")
-                continue
-            func_min = sub_funcs[func]  # now here we have to assume nobody is using any of our substitute vars
-            # should be a pretty safe assumption but knowing for sure would require about the same amount of effort
-            # as it would to replace all user defined variables
-            ucode = ""  # this is rather hacky
-            sections = [0]
-            for m in re.finditer(r"(?<!\.)%s\(" % func, mmcode):
-                sections.append(m.span()[0])
-                sections.append(m.span()[1])
-            sections.append(len(mmcode))
-            i = 2       # change rather to very
-            while i < len(sections):
-                ucode += mmcode[sections[i-2]:sections[i-1]] + func_min + '('
-                i += 2
-            ucode += mmcode[sections[i-2]:]
-
-            ucode = f"{func_min}={func}\n" + ucode
-            mmcode = ucode
-            # a space isn't any shorter than \n so why not use \n
-
-    for string, count in str_reuse.items():
-        if count >= 2:
-            # we can't auto replace strings without a full parser
-            # unlike with the stdlib functions we cant make a lookup table ahead of time
-            # and generating shorter names on the fly sounds like an absolute nightmare no thanks
-            print(f'Warning: string "{string}" of len {len(string)} reused {count} times')
-
-    return requires + mmcode.strip()
+    return requires + mmcode.strip().replace('\n', ' ')
 
 
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description="Minify tsv3 scripts, useful for embedding",
-                                     formatter_class=argparse.RawTextHelpFormatter)
-    parser.add_argument("source", type=str, nargs='+', help="source files to minify")
-    parser.add_argument("-d", type=str, nargs='?', help="destination folder for minified scripts"
-                                                        "\ndefault: ./", default='./')
-    parser.add_argument("--replace-functions", action="store_true", default=False,
-                        help="automatically replace reused functions instead of just warning\ndefault: false")
+    argparser = argparse.ArgumentParser(description="Minify tsv3 scripts, useful for embedding",
+                                        formatter_class=argparse.RawTextHelpFormatter)
+    argparser.add_argument("source", type=str, nargs='+', help="source files to minify")
+    argparser.add_argument("-d", type=str, nargs='?', help="destination folder for minified scripts"
+                                                           "\ndefault: .", default='.')
+    argparser.add_argument("--auto-replace", action="store_true", default=False,
+                           help="automatically replace reused functions, variables and strings instead of just warning\n"
+                                "and attempt to generate shorter names for reused variables \ndefault: false")
+    argparser.add_argument("-v", action="store_true", default=False,
+                           help="prints even more information to the console than usual")
+    argparser.add_argument("--such-meme", action="store_true", default=False,
+                           help="replaces destination file if it already exists \ndefault: false")
 
-    args = parser.parse_args()
+    args = argparser.parse_args()
     files = args.source
-    dest = args.d[:-1] if args.d[-1] == '/' else args.d
-    replace_functions = args.replace_functions if args.replace_functions is not None else False
+    dest = args.d
+    auto_replace = args.auto_replace
+    verbose = "INFO" if args.v else "WARNING"
+    logging.basicConfig(level=verbose, format="{message}", style='{')
 
+    print(f"Automatic replacement: {'ENABLED' if auto_replace else 'DISABLED'}")
     for file in files:
-        print(f"Minifying {file}")
+        print(f"\nMinifying {file}")
         with open(file, 'r') as f:
-            r = minify(f.read())
-        file = file.split(sep='.')[0].split(sep='/')[-1]
-        if dest != '.':
-            f = open(f"{dest}/{file}.te", 'w')
+            logging.info("Stripping comments and whitespace (pass 1)")
+            r = whitespacent(f.read())
+            logging.info("Parsing file")
+            r = parser(r)
+            logging.info("Searching for optimisations")
+            r = minify(r[0], r[1], r[2])
+            logging.info("Stripping whitespace (pass 2)")
+            r = whitespacent(r)
+        file = path.splitext(path.basename(file))[0]
+        if path.exists(path.join(dest, f"{file}.te")) and not args.such_meme:
+            f = open(path.join(dest, f"{file}_min.te"), 'w')
         else:
-            f = open(f"{dest}/{file}_min.te", 'w')
+            f = open(path.join(dest, f"{file}.te"), 'w')
+        logging.info(f"Writing to {f.name}")
         f.write(r)
+        print("Done!")
+        f.close()