Bit wiggling inside some Firmware. 0xHENNE :P

SenaBitWiggler/build.gradle

@@ -0,0 +1,33 @@
+defaultTasks 'distZip'
+
+apply plugin: 'java'
+apply plugin: 'application'
+
+
+sourceCompatibility = 1.8
+version = 'SNAPSHOT'
+mainClassName = 'de.nplusc.izc.senabitwiggler.EntryPoint'
+
+
+jar{
+	manifest{
+		attributes 'Implementation-Title': 'senabitwiggler',
+					'Implementation-Version': 'SNAPSHOT',
+					'Main-Class': 'de.nplusc.izc.senabitwiggler.EntryPoint'
+					
+	}
+}
+
+repositories{
+	mavenCentral()
+}
+
+dependencies{
+	compile group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.+'
+	compile group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.+'
+	compile group: 'org.apache.logging.log4j', name: 'log4j-iostreams', version: '2.+'
+    compile group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: '2.+'
+    compile group: 'org.apache.logging.log4j', name: 'log4j-jul', version: '2.+'
+    compile "org.yaml:snakeyaml:1.14"
+    compile("com.google.guava:guava:31.0.1-jre")
+}

SenaBitWiggler/src/main/java/de/nplusc/izc/senabitwiggler/EntryPoint.java

@@ -0,0 +1,352 @@
+package de.nplusc.izc.senabitwiggler;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Strings;
+import com.google.common.primitives.Longs;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.*;
+import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+
+public class EntryPoint
+{
+    private static Logger l = LogManager.getLogger();
+    public static void main(String[] args)
+    {
+        if(args.length!=4)
+        {
+            printUsage();
+            return;
+        }
+        boolean extract = args[0].equalsIgnoreCase("extract");
+        boolean immport = args[0].equalsIgnoreCase("import");
+
+        if(!(extract||immport))
+        {
+            printUsage();
+            return;
+        }
+
+        boolean shorty = args[1].equalsIgnoreCase("short");
+        boolean longy = args[1].equalsIgnoreCase("long");
+
+        if(!(shorty||longy))
+        {
+            printUsage();
+            return;
+        }
+
+        if(shorty)
+        {
+            System.out.println("Need to debug the format more first. Aborting now");
+            return;
+        }
+
+        File input = new File(args[2]);
+        if(immport&&input.exists())
+        {
+            System.out.println("Refusing to overwrite a existing firmware archive. Use a fresh name instead");
+            return;
+        }
+        if(extract&&!input.exists())
+        {
+            System.out.println("Can't extract thin air. Give a file please");
+        }
+
+        if(extract)
+        {
+            if(longy)
+            {
+                try {
+                    extractFirmwareLong(input,args[3]);
+                } catch (InputInvalidException e) {
+                    System.out.println("Zarf! File was bad");
+                    e.printStackTrace();
+                }
+            }
+        }
+        else
+        {
+            if(immport)
+            {
+                if(longy)
+                {
+                    assembleFirmware(input,args[3]);
+                }
+            }
+        }
+    }
+
+    private static void assembleFirmware(File output, String inputfolder)
+    {
+        Yaml y = new Yaml();
+        try {
+            LongHeader h = y.loadAs(new FileReader(new File(inputfolder+File.separator+"header.yml")),LongHeader.class);
+            RandomAccessFile f = new RandomAccessFile(output,"rw");
+            f.write(h.getVersion_raw());
+            f.write(LongToRawBytes(h.getMagicShit()));
+            f.write(LongToRawBytes(h.getRandom_id()));
+            HeaderRecord[] records = h.getHeaderRecords();
+            int count = records.length;
+            f.write(LongToRawBytes(count));
+            long offset = records[0].getOffset();
+            for(int i=0;i<count;i++)
+            {
+                HeaderRecord r = records[i];
+                File handle = new File(inputfolder+File.separator+r.getFilename());
+                r.setLength(handle.length());
+                r.setOffset(offset+0);
+                offset+=handle.length();
+
+                RandomAccessFile file = new RandomAccessFile(handle,"r");
+                byte[] bfr = new byte[(int)handle.length()];
+                file.read(bfr);
+                byte[] newmd5 = MessageDigest.getInstance("MD5").digest(bfr);
+                r.setMd5sum(newmd5);
+            }
+
+            for(int i=0;i<count;i++)
+            {
+                HeaderRecord r = records[i];
+                f.write(LongToRawBytes(r.getShortflag_1()));
+                f.write(LongToRawBytes(r.getShortflag_2()));
+                f.write(LongToRawBytes(r.getOffset()));
+                f.write(LongToRawBytes(r.getLength()));
+                f.write(r.getMd5sum());
+            }
+
+            for(int i=0;i<count;i++)
+            {
+                HeaderRecord r = records[i];
+                f.write(LongToRawBytes(r.getShortflag_1()));
+                f.write(LongToRawBytes(r.getShortflag_2()));
+                f.write(LongToRawBytes(r.getOffset()));
+                f.write(LongToRawBytes(r.getLength()));
+                f.write(LongToRawBytes(r.getFlag_1()));
+                f.write(LongToRawBytes(r.getFlag_2()));
+                f.write(LongToRawBytes(r.getFlag_3()));
+                f.write(LongToRawBytes(r.getFlag_4()));
+                f.write(LongToRawBytes(r.getFlag_5()));
+                f.write(LongToRawBytes(r.getUnknown_id()));
+                f.write(r.getPadding());
+
+                byte[] filenamebytes = r.getFilename().getBytes(Charsets.US_ASCII);
+                byte[] filename = new byte[128];
+                for(int j=0;j<128;j++)
+                {
+                    if(j<filenamebytes.length)
+                    {
+                        filename[j]=filenamebytes[j];
+                    }
+                    else
+                    {
+                        filename[j]=0;
+                    }
+                }
+                f.write(filename);
+                f.write(r.getMd5sum());
+            }
+
+            for(int i=0;i<count;i++)
+            {
+                HeaderRecord r = records[i];
+                File handle = new File(inputfolder+File.separator+r.getFilename());
+                RandomAccessFile file = new RandomAccessFile(handle,"r");
+                byte[] bfr = new byte[(int)handle.length()];
+                file.read(bfr);
+                f.write(bfr);
+            }
+            f.close();
+
+        } catch (FileNotFoundException e) {
+            e.printStackTrace();
+        } catch (IOException e) {
+            e.printStackTrace();
+        } catch (NoSuchAlgorithmException e) {
+            e.printStackTrace();
+        }
+    }
+
+
+    private static byte[] LongToRawBytes(long l)
+    {
+        byte[] tmp = Longs.toByteArray(l);
+        return new byte[]{tmp[7],tmp[6],tmp[5],tmp[4]};
+    }
+
+
+
+    private static void extractFirmwareLong(File input, String output) throws InputInvalidException {
+        byte filler = 0x00;
+        try (RandomAccessFile f = new RandomAccessFile(input,"r")) {
+            LongHeader hdr = new LongHeader();
+            byte[] headermagic = new byte[24];
+            f.read(headermagic);
+            String versionName = new String(headermagic, Charsets.US_ASCII).split("\0")[0]; //hack, can be done since strings are shorty. https://stackoverflow.com/a/8843313/1405227
+            hdr.setVersion_raw(headermagic);
+            hdr.setVersion(versionName);
+            //12 more bytes
+            byte[] magic = new byte[4];
+            f.read(magic);
+
+            hdr.setMagicShit(Longs.fromBytes(filler,filler,filler,filler,magic[3],magic[2],magic[1],magic[0]));
+
+            byte[] unknown_id = new byte[4];
+
+            f.read(unknown_id);
+
+            hdr.setRandom_id(Longs.fromBytes(filler,filler,filler,filler,unknown_id[3],unknown_id[2],unknown_id[1],unknown_id[0]));
+
+            byte[] count_raw = new byte[4];
+            f.read(count_raw);
+
+            long count_l = (Longs.fromBytes(filler,filler,filler,filler,count_raw[3],count_raw[2],count_raw[1],count_raw[0]));
+
+            if(count_l>Integer.MAX_VALUE)
+            {
+                throw new InputInvalidException();
+            }
+            int count = (int)count_l;
+            HeaderRecord[] records = new HeaderRecord[count];
+            byte[] skip = new byte[count*32]; /*skipping over shorty header, repeats content with long one*/
+            f.read(skip);
+            for(int i=0;i<count;i++)
+            {
+                HeaderRecord record = new HeaderRecord();
+
+
+                byte[] flags1 = new byte[4];
+                f.read(flags1);
+
+                record.setShortflag_1(Longs.fromBytes(filler,filler,filler,filler,flags1[3],flags1[2],flags1[1],flags1[0]));
+
+                byte[] flags2 = new byte[4];
+                f.read(flags2);
+
+                record.setShortflag_2(Longs.fromBytes(filler,filler,filler,filler,flags2[3],flags2[2],flags2[1],flags2[0]));
+
+                byte[] offset = new byte[4];
+                f.read(offset);
+
+                record.setOffset(Longs.fromBytes(filler,filler,filler,filler,offset[3],offset[2],offset[1],offset[0]));
+
+                byte[] length = new byte[4];
+                f.read(length);
+
+                record.setLength(Longs.fromBytes(filler,filler,filler,filler,length[3],length[2],length[1],length[0]));
+
+
+
+                long[] flags = new long[5];
+                for(int j=0;j<5;j++)
+                {
+                    byte[] flags_raw = new byte[4];
+                    f.read(flags_raw);
+
+                    flags[j] = (Longs.fromBytes(filler,filler,filler,filler,flags_raw[3],flags_raw[2],flags_raw[1],flags_raw[0]));
+                }
+
+                record.setFlag_1(flags[0]);
+                record.setFlag_2(flags[1]);
+                record.setFlag_3(flags[2]);
+                record.setFlag_4(flags[3]);
+                record.setFlag_5(flags[4]);
+
+                byte[] unknown = new byte[4];
+                f.read(unknown);
+
+                record.setUnknown_id(Longs.fromBytes(filler,filler,filler,filler,unknown[3],unknown[2],unknown[1],unknown[0]));
+
+                byte[] padding = new byte[128];
+
+                f.read(padding);
+
+                record.setPadding(padding);
+
+                byte[] filename_raw = new byte[128];
+
+                f.read(filename_raw);
+
+                String filename = new String(filename_raw, Charsets.US_ASCII).split("\0")[0]; //hack, can be done since strings are shorty. https://stackoverflow.com/a/8843313/1405227
+                record.setFilename(filename);
+
+                byte[] md5 = new byte[16];
+
+                f.read(md5);
+
+                record.setMd5sum(md5);
+
+                records[i]=record;
+
+            }
+            hdr.setHeaderRecords(records);
+            Yaml y = new Yaml();
+
+            y.dump(hdr, new FileWriter(new File(output+File.separator+"header.yml")));
+
+            for(int i=0;i<records.length;i++)
+            {
+                HeaderRecord r = records[i];
+                String target = output+File.separator+r.getFilename();
+                long offset = r.getOffset();
+                int len = (int)r.getLength();
+                byte[] file = new byte[len];
+                f.seek(offset);
+                f.read(file);
+                RandomAccessFile out = new RandomAccessFile(target,"rw");
+                out.write(file);
+                out.close();
+
+            }
+
+        } catch (FileNotFoundException e) {
+            e.printStackTrace();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+
+
+    private static void printUsage()
+    {
+        System.out.println("Usage:");
+        System.out.println("program DIRECTION VARIANT FILE FOLDER");
+        System.out.println("DIRECTION: extract OR import");
+        System.out.println("Variant: short OR long");
+        System.out.println("only long supported so far. short header needs more information");
+
+    }
+
+
+
+
+    private static final char[] HEX_ARRAY = "0123456789ABCDEF".toCharArray();
+    public static String bytesToHex(byte[] bytes) {
+        char[] hexChars = new char[bytes.length * 2];
+        for (int j = 0; j < bytes.length; j++) {
+            int v = bytes[j] & 0xFF;
+            hexChars[j * 2] = HEX_ARRAY[v >>> 4];
+            hexChars[j * 2 + 1] = HEX_ARRAY[v & 0x0F];
+        }
+        return new String(hexChars);
+    }
+}
+
+
+// http://www.tinyosshop.com/download/ADK_CSR867x.WIN4.3.1.5.zip für die tools
+
+// C:\ADK_CSR867x.WIN4.3.1.5\tools\bin\XUV2BIN.exe -e vp.bin vp.xuv
+
+//C:\ADK_CSR867x.WIN4.3.1.5\tools\bin\unpackfile.exe vp.xuv out
+// resultat: out mit prm-dateien. bei SRL2 sind das raw-PCMs, Mono, 16bit 8khz. leider keine header zum rumverzinken....
+// C:\ADK_CSR867x.WIN4.3.1.5\tools\bin\packfile.exe out vp2.xuv
+// C:\ADK_CSR867x.WIN4.3.1.5\tools\bin\XUV2BIN.exe -d vp2.xuv vp2.bin
+// danach die vp.bin mit vp2.bin austauschen und repacken

SenaBitWiggler/src/main/java/de/nplusc/izc/senabitwiggler/HeaderRecord.java

@@ -0,0 +1,152 @@
+package de.nplusc.izc.senabitwiggler;
+
+public class HeaderRecord {
+    /*Short Header and Long Header*/
+    /*16 bytes*/
+    private long shortflag_1;
+    private long shortflag_2;
+    private long offset;
+    private long length;
+    private byte[] md5sum;
+
+
+
+
+    // Long Header Only
+    /*Has to stay unchanged, saving it into the yml*/
+    /*
+    shortflag_1
+    shortflag_2
+    offset
+    length
+    */
+    private long flag_1;
+    private long flag_2;
+    private long flag_3;
+    private long flag_4;
+    private long flag_5;
+
+    private long unknown_id;
+    /*128 bytes*/
+    private byte[] padding;
+    private String filename;
+    /*filename padded to 128 bytes*/
+    /*md5sum*/
+
+
+    /*Serializer stuff for snakeyaml*/
+
+    public long getShortflag_1() {
+        return shortflag_1;
+    }
+
+    public void setShortflag_1(long shortflag_1) {
+        this.shortflag_1 = shortflag_1;
+    }
+
+    public long getShortflag_2() {
+        return shortflag_2;
+    }
+
+    public void setShortflag_2(long shortflag_2) {
+        this.shortflag_2 = shortflag_2;
+    }
+
+    public long getOffset() {
+        return offset;
+    }
+
+    public void setOffset(long offset) {
+        this.offset = offset;
+    }
+
+    public long getLength() {
+        return length;
+    }
+
+    public void setLength(long length) {
+        this.length = length;
+    }
+
+    public byte[] getMd5sum() {
+        return md5sum;
+    }
+
+    public void setMd5sum(byte[] md5sum) {
+        this.md5sum = md5sum;
+    }
+
+    public long getFlag_1() {
+        return flag_1;
+    }
+
+    public void setFlag_1(long flag_1) {
+        this.flag_1 = flag_1;
+    }
+
+    public long getFlag_2() {
+        return flag_2;
+    }
+
+    public void setFlag_2(long flag_2) {
+        this.flag_2 = flag_2;
+    }
+
+    public long getFlag_3() {
+        return flag_3;
+    }
+
+    public void setFlag_3(long flag_3) {
+        this.flag_3 = flag_3;
+    }
+
+    public long getFlag_4() {
+        return flag_4;
+    }
+
+    public void setFlag_4(long flag_4) {
+        this.flag_4 = flag_4;
+    }
+
+    public long getFlag_5() {
+        return flag_5;
+    }
+
+    public void setFlag_5(long flag_5) {
+        this.flag_5 = flag_5;
+    }
+
+    public long getUnknown_id() {
+        return unknown_id;
+    }
+
+    public void setUnknown_id(long unknown_id) {
+        this.unknown_id = unknown_id;
+    }
+
+    public byte[] getPadding() {
+        return padding;
+    }
+
+    public void setPadding(byte[] padding) {
+        this.padding = padding;
+    }
+
+    public String getFilename() {
+        return filename;
+    }
+
+    public void setFilename(String filename) {
+        this.filename = filename;
+    }
+
+
+    public void setMd5String(String md5)
+    {
+        //HACK, NOP
+    }
+    public String getMd5String()
+    {
+        return EntryPoint.bytesToHex(md5sum);
+    }
+}

SenaBitWiggler/src/main/java/de/nplusc/izc/senabitwiggler/InputInvalidException.java

@@ -0,0 +1,4 @@
+package de.nplusc.izc.senabitwiggler;
+
+public class InputInvalidException extends Exception{
+}

SenaBitWiggler/src/main/java/de/nplusc/izc/senabitwiggler/LongHeader.java

@@ -0,0 +1,59 @@
+package de.nplusc.izc.senabitwiggler;
+
+public class LongHeader {
+
+    /*assuming 24 bytes but going to play it safe and storing the excess bytes, too*/
+    private String version;
+    private byte[] version_raw;
+
+    private long MagicShit; /*0x19 0x28 0xc5 0xE6*/
+    /*related to the random_id in the headers
+    read as BCD here and then that number as hex in the file. some variants got mangled IDs, no rule so far.
+     */
+    private long random_id;
+
+    /*long count, not needed since the records[] tells that count*/
+
+    private HeaderRecord[] headerRecords;
+
+
+    public String getVersion() {
+        return version;
+    }
+
+    public void setVersion(String version) {
+        this.version = version;
+    }
+
+    public byte[] getVersion_raw() {
+        return version_raw;
+    }
+
+    public void setVersion_raw(byte[] version_raw) {
+        this.version_raw = version_raw;
+    }
+
+    public long getMagicShit() {
+        return MagicShit;
+    }
+
+    public void setMagicShit(long magicShit) {
+        MagicShit = magicShit;
+    }
+
+    public long getRandom_id() {
+        return random_id;
+    }
+
+    public void setRandom_id(long random_id) {
+        this.random_id = random_id;
+    }
+
+    public HeaderRecord[] getHeaderRecords() {
+        return headerRecords;
+    }
+
+    public void setHeaderRecords(HeaderRecord[] headerRecords) {
+        this.headerRecords = headerRecords;
+    }
+}

SenaBitWiggler/src/main/resources/log4j2.xml

@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="WARN">
+    <Appenders>
+        <Console name="Console" target="SYSTEM_OUT">
+            <PatternLayout pattern="%highlight{%d{HH:mm:ss.SSS} [%t] %-5level %l - %msg%n}{FATAL=red, ERROR=red, WARN=yellow, INFO=green, DEBUG=blue, TRACE=blue}"/>
+        </Console>
+        
+        <Console name="ConsoleExternal" target="SYSTEM_OUT">
+            <PatternLayout pattern="%highlight{%d{HH:mm:ss.SSS} [[%logger]] %-5level - %msg%n}{FATAL=red, ERROR=red, WARN=yellow, INFO=green, DEBUG=blue, TRACE=blue}"/>
+        </Console>
+        
+        
+        <RollingFile name="logfile" fileName="izpl.log" filePattern="izpl-%i.log">
+        <Policies>
+            <OnStartupTriggeringPolicy />
+            </Policies>
+            <DefaultRolloverStrategy min="1" max="1"/>
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %l - %msg%n"/>
+        </RollingFile>   
+
+        <RollingFile name="logfileExternal" fileName="izpl.log" filePattern="izpl-%i.log">
+        <Policies>
+            <OnStartupTriggeringPolicy />
+            </Policies>
+            <DefaultRolloverStrategy min="1" max="1"/>
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} [[%logger]] %-5level - %msg%n"/>
+        </RollingFile>   
+        
+    </Appenders>
+    <Loggers>
+    <Root level="info">
+        <AppenderRef ref="Console"/>
+        <AppenderRef ref="logfile"/>
+    </Root>
+    <Logger name="External" level="info" additivity="false">
+        <AppenderRef ref="ConsoleExternal"/>
+        <AppenderRef ref="logfileExternal"/>
+    </Logger>
+    <Logger name="uk.co.caprica.vlcj" level="info" additivity="false">
+        <AppenderRef ref="ConsoleExternal"/>
+        <AppenderRef ref="logfileExternal"/>
+    </Logger>
+    <Logger name="net.java" level="info" additivity="false">
+        <AppenderRef ref="ConsoleExternal"/>
+        <AppenderRef ref="logfileExternal"/>
+    </Logger>
+    </Loggers>
+</Configuration>

settings.gradle

@@ -1,4 +1,4 @@
 include 'ToolKit','iZpl',  'IZSetup','WPCMGr','UpidTK', 'izstreamer', 'LogBlockHeatMapper','iZlaunch',
 'iZpaqSFX','MazeViewer','TWPUtil',"iZplPlugins:WMP","iZplPlugins:foobar2000_others","iZplPlugins:itunes","iZplPlugins:GameRadio",
 "iZplPlugins:Editor","izpl-shared","iZpl-server","iZplPlugins:Smartphonizer","QuickStuff","external:java-progressbar","iZplPlugins:rtsslink","iZplPlugins:JukeBox",
-"iZplPlugins:Extractor","spungit_deathtimer","iZplPlugins:discordjukebox"
+"iZplPlugins:Extractor","spungit_deathtimer","iZplPlugins:discordjukebox","SenaBitWiggler"