resync of crunchers

QuickStuff/build.gradle

@@ -7,4 +7,5 @@ dependencies{
     }
     implementation 'org.bouncycastle:bcprov-ext-jdk18on:1.71'
     implementation 'com.fazecast:jSerialComm:[2.0.0,3.0.0)'
-}
+    implementation 'com.eclipsesource.minimal-json:minimal-json:0.9.5'
+}

QuickStuff/src/main/java/QuickVerifyCrap/EVS_CTR.java

@@ -0,0 +1,77 @@
+package QuickVerifyCrap;
+
+import com.eclipsesource.json.*;
+
+import java.io.*;
+
+
+public class EVS_CTR {
+    static boolean switcheroo = true;
+    public static void main(String[] args) throws Exception{
+
+        if(switcheroo)
+        {
+            JsonObject obj = Json.parse(new FileReader("C:\\lmaa\\placeDE-exceling\\in.json")).asObject();
+            JsonArray arr = obj.get("messages").asArray();
+            PrintWriter w = new PrintWriter(new FileWriter("C:\\lmaa\\placeDE-exceling\\synthetic.csv"));
+            w.println("type;author;length;cookiecount;timestamp");
+            for (int i = 0; i < arr.size(); i++)
+            {
+                boolean isEventSys = false;
+                JsonObject msg = arr.get(i).asObject();
+                JsonObject author = msg.get("author").asObject();
+                String id = author.getString("id","---");
+                if(id.equals("958372977714540574"))
+                {
+                    isEventSys=true;
+                    System.out.println("Eventsys");
+                }
+
+                String message = msg.getString("content","");
+
+                int len = message.length();
+                String timestamp = msg.getString("timestamp","").substring(0,19);
+                int cookiecount = 0;
+                String type = "default";
+                if(isEventSys)
+                {
+                    type="leaderboard";
+                    JsonArray embeds = msg.get("embeds").asArray();
+                    if(!(embeds.size()==0))
+                    {
+                        JsonObject embed = embeds.get(0).asObject();
+                        String title = embed.getString("title","");
+                        if(title.equals("Gesammelte Cookies"))
+                        {
+                            type="cookie";
+                            String description = embed.getString("description","0");
+                            String cntS = description.replace("Ihr konntet ","").replace(" Cookies sammeln.","").replace(" Cookie sammeln.","");
+                            cookiecount = Integer.valueOf(cntS);
+                        }
+                    }
+                }
+                w.println(type+";"+id+";"+len+";"+cookiecount+";"+timestamp);
+            }
+            w.close();
+        }
+        else
+        {
+            BufferedReader r = new BufferedReader(new FileReader("C:\\lmaa\\placeDE-exceling\\in2.csv"));
+            int lns=0;
+            String line = r.readLine();
+            while(line!=null)
+            {
+                if(line.contains(",\"EventSystem#0907\""))
+                {
+                    System.out.println(lns+"|"+line);
+                    lns=0;
+                }
+                else
+                {
+                    lns++;
+                }
+                line=r.readLine();
+            }
+        }
+    }
+}

QuickStuff/src/main/java/QuickVerifyCrap/Lazy.java

@@ -11,7 +11,7 @@ public class Lazy {
             System.out.print(" ");
         }*/
         SplitSecondArkSplitter.HACK=true;
-        SplitSecondArkSplitter.main(null);
+        //SplitSecondArkSplitter.main(null);
         SplitSecondArkSplitter.HACK=false;
         sweep("O:\\SplitSecond\\data");
 
@@ -23,10 +23,18 @@ public class Lazy {
     {
         File f = new File(path);
         for (File file : f.listFiles()) {
-            if(file.getName().endsWith(".ark"))
+            if(file.getName().endsWith(".ark")) // &&file.getName().contains("resident")
             {
                 //System.out.println(file.getAbsolutePath());
                 SplitSecondArkSplitter.main(new String[]{file.getAbsolutePath()});
+                System.out.println("Ark filed");
+                System.out.println(file.getName());
+            }
+            else
+            {
+                //System.out.println("fake filed");
+                //System.out.println(file.getName());
+                //
             }
             if(file.isDirectory())
             {

QuickStuff/src/main/java/QuickVerifyCrap/SplitSecondArkSplitter.java

@@ -2,31 +2,68 @@ package QuickVerifyCrap;
 
 import java.io.*;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 
 public class SplitSecondArkSplitter {
     public static boolean HACK = true;
 
+
+    public static void wiggler(String line, HashMap<String,String> filehashes, PrintWriter bw)
+    {
+        long wiggled = SplitSecondChecksumCheck.wiggleChecksum(line.replace('/','\\'));
+        filehashes.put(wiggled + "", line);
+
+        if(HACK)
+            bw.println(Long.toHexString(wiggled) + "|" + line);
+
+        wiggled = SplitSecondChecksumCheck.wiggleChecksum(line.replace('\\','/'));
+        filehashes.put(wiggled + "", line);
+        wiggled = SplitSecondChecksumCheck.wiggleChecksum(line);
+        filehashes.put(wiggled + "", line);
+        if(HACK)
+            bw.println(Long.toHexString(wiggled) + "|" + line);
+    }
     public static void main(String[] args) {
         HashMap<String, String> filehashes = new HashMap<>();
+        HashSet<String> filehashes_scratchoff = new HashSet<>();
         try (PrintWriter bw = new PrintWriter(new FileWriter("D:\\LOA\\000014\\filenames_debugdump.txt",true))) {
 
             try (BufferedReader br = new BufferedReader(new FileReader("D:\\LOA\\000014\\filenames_preprocessed.txt"))) {
                 br.lines().forEach((line) ->
                 {
-                    long wiggled = SplitSecondChecksumCheck.wiggleChecksum(line);
-                    filehashes.put(wiggled + "", line);
-                    if(HACK)
-                        bw.println(Long.toHexString(wiggled) + "|" + line);
+                    wiggler(line,filehashes,bw);
+
+                    if(line.contains("PhysicsInstances"))
+                    {
+                        String[] hackificatedLine = line.replace('\\','/').replace('/','\\').split("\\\\");
+                        int len = hackificatedLine.length;
+                        String replaceme = hackificatedLine[len-2]+"_damage";
+                        hackificatedLine[len-1] = hackificatedLine[len-1].replace(hackificatedLine[len-2],replaceme);
+                        hackificatedLine[len-2]=replaceme;
+                        wiggler(String.join("/",hackificatedLine),filehashes,bw);
+                    }
+                    if(line.contains("Vehicles"))
+                    {
+                        wiggler(line.replace("Low","XLow"),filehashes,bw);
+                    }
+                    /*if(line.contains("\\"))
+                    {
+                        String line_looper = line.split("\\\\",1)[1];
+                        {
+
+                        }
+                    }*/
+
                 });
             } catch (FileNotFoundException e) {
                 e.printStackTrace();
             } catch (IOException e) {
                 e.printStackTrace();
             }
-            if (HACK)
-                return;
+            /*if (HACK)
+                return;*/
             String[] levelbases = {"airport_test_03", "docks", "downtown", "end_sequence",
                     "graveyard", "nem_downtown", "nem_graveyard", "nem_storm", "nem_training", "nem_warehouse", "powerplant"};
 
@@ -37,6 +74,7 @@ public class SplitSecondArkSplitter {
                 for (int j = 0; j < 5; j++) {
                     filehashes.put(SplitSecondChecksumCheck.wiggleChecksum("q" + prefix + "_rig" + j + "MD.model.stream") + "", "q" + prefix + "_rig" + j + "MD.model.stream");
                     filehashes.put(SplitSecondChecksumCheck.wiggleChecksum("q" + prefix + "_rig" + j + "L.model.stream") + "", "q" + prefix + "_rig" + j + "L.model.stream");
+                    filehashes.put(SplitSecondChecksumCheck.wiggleChecksum("q" + prefix + "_rig" + j + "S.model.stream") + "", "q" + prefix + "_rig" + j + "S.model.stream");
                 }
 
                 for (String base : levelbases) {
@@ -44,7 +82,12 @@ public class SplitSecondArkSplitter {
                 }
             }
 
+            filehashes.forEach((k,v)->filehashes_scratchoff.add(v));
+
             try {
+                String arkbasename = new File(args[0]).getName();
+                PrintWriter w = new PrintWriter("Q:\\LOA\\000014\\result\\" + arkbasename+".lst");
+                final PrintWriter wdb = new PrintWriter("Q:\\LOA\\000014\\result\\" + arkbasename+".lst.dbg");
                 RandomAccessFile f = new RandomAccessFile(args[0], "r");
                 f.readInt();
                 f.readInt();
@@ -73,16 +116,38 @@ public class SplitSecondArkSplitter {
                     System.out.println(Long.toHexString(csum2));
                     fmd.filename = filename != null ? filename : "UNKNOWN_" + csum2;
                     fmd.checksum = csum2;
+                    if(filename !=null)
+                    {
+                        System.out.println("Matched:"+filename);
+                        String fn1 = filename.replace('/','\\');
+                        final String fn2 = fn1.replace('\\','/');
+                        filehashes_scratchoff.removeIf((str)->{
+                            String ifn1 = str.replace('/','\\');
+                            String ifn2 = ifn1.replace('\\','/');
+                            return fn2.equals(ifn2);
+                        });
+                    }
                     fileslst.add(fmd);
                 }
+                filehashes_scratchoff.forEach((itm)->{wdb.println(itm);});
+                wdb.close();
                 for (FileMetaData fmd : fileslst) {
-                    String arkbasename = new File(args[0]).getName();
                     String realresult = "Q:\\LOA\\000014\\result\\" + arkbasename + "\\" + fmd.filename;
                     File dst = new File(realresult);
                     dst.getParentFile().mkdirs();
                     bw.println(">>"+arkbasename+"||"+Long.toHexString(fmd.checksum)+"||"+fmd.filename);
-                    if (fmd.zipped) {
-
+                    w.println(Long.toHexString(fmd.checksum)+";"+fmd.filename);
+                    if(fmd!=null)
+                    {
+                        continue;
+                    }
+                    if (fmd.zipped&&!fmd.zipped) {
+                        String transferfile = "Q:\\LOA\\000014\\scratch\\result.dat";
+                        File tmp = new File(transferfile);
+                        if(tmp.exists())
+                        {
+                            tmp.delete();
+                        }
                         String outfile = "Q:\\LOA\\000014\\scratch\\out.dat";
                         File of = new File(outfile);
                         of.getParentFile().mkdirs();
@@ -96,9 +161,9 @@ public class SplitSecondArkSplitter {
                         rof.writeInt(size);
                         rof.write(compressed);
                         rof.close();
-                        File workdir = new File("D:\\LOA\\000014\\scratch");
+                        File workdir = new File("Q:\\LOA\\000014\\scratch");
                         runTool(workdir, "D:\\LOA\\000014\\quickbms.exe", "-R", "D:\\LOA\\000014\\helper.bms", outfile);
-                        String transferfile = "D:\\LOA\\000014\\scratch\\result.dat";
+
                         new File(transferfile).renameTo(dst);
                     } else {
                         byte[] compressed = new byte[fmd.zsize];
@@ -109,6 +174,7 @@ public class SplitSecondArkSplitter {
                         rof.close();
                     }
                 }
+                w.close();
             } catch (FileNotFoundException e) {
                 e.printStackTrace();
             } catch (IOException e) {

QuickStuff/src/main/java/QuickVerifyCrap/SplitSecondChecksumCheck.java

@@ -19,7 +19,7 @@ public class SplitSecondChecksumCheck {
         long csum = 0;
         for(byte c:raw)
         {
-            //long massagedChar = massage_character(c&0xFF);
+            long massagedChar = massage_character(c);
             //System.out.println(c);
             csum =csum*0x1f+c;
             csum = csum&0xFFFFFFFFL;