reworked WPCMGr for multithreading and a more usable progress bar. IZPL fix on zero multiplier

WPCMGr: Changed Command assembly from immediate execute to queued execute.
Bframes per package only added if package got changed files.
Queue processed with multiple threads.
Progressbar more uisable since queue only got the executed actions and not the skipped ones

ToolKit/src/main/java/de/nplusc/izc/tools/IOtools/FileTK.java

@@ -572,10 +572,12 @@ public class FileTK
     {
         try
         {
+            l.trace("COPY({},{})",in,out);
             Files.copy(in.toPath(), out.toPath());
         }
         catch (IOException ex)
         {
+            l.error(ex);
             l.error("Kopieren fehlgeschlagen");
         }
     }

WPCMGr/src/main/java/de/nplusc/izc/Utilities/WPCMgr/Synchronizer.java

@@ -31,7 +31,9 @@ import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Queue;
 import java.util.Set;
+import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import org.apache.commons.cli.ParseException;
@@ -105,7 +107,7 @@ public class Synchronizer
             for(String id:profiles)
             {
                 l.info("Converting Profile {} ({}/{})",id,i,count);
-                new Synchronizer().handleWPC(targetCache, debugmode, dryRun,id); 
+                new Synchronizer().handleWPC(targetCache, debugmode, dryRun, id); 
                 i++;
             }
         }
@@ -223,6 +225,12 @@ public class Synchronizer
         final LinkedHashMap<String, Object> data = (LinkedHashMap<String, Object>) WPCUtils.getConfig(cachedir);
         WPCUtils.ensureProfileExists(data, profile);
         imagemagickPath=(String) data.get("imagemagick");
+        if(!data.containsKey("threads"))
+        {
+            l.warn("No thread count configured, falling back to default (1)");
+            data.put("threads",1);
+        }
+        int threads =(int) data.get("threads");
         if(imagemagickPath==null)
         {
             l.error("No ImageMagick path given");
@@ -304,7 +312,7 @@ public class Synchronizer
         String[] dirsExist = FileTK.getDirectoryContent(cachedir, true);
         //final int pid[] = new int[]{0};
         progressBar = ConsoleProgressBar.on(System.out)
-                .withFormat("[:bar] :percent% :elapsed/:total :msg")
+                .withFormat("[:bar] :percent% :elapsed/:total (:msg)")
                 .withTotalSteps(dirsExist.length);
         progressBar.start();
         progressBar.setMessage("Detecting Files...");
@@ -375,15 +383,12 @@ public class Synchronizer
             progressBar.tickOne();
         });
         l.trace("finalize pre");
+        System.out.println();
         //progressBar.complete();
         
         identifyAll(theState);
         
-        
-        progressBar = progressBar.withTotalSteps(totalFiles + 1);
-        progressBar.setMessage("Dateien:0/" + totalFiles);
-        progressBar.start();
-        currentFile=1;
+        List<String[]> commands = new ArrayList<String[]>();
         //progressBar.tickOne();
         existingPKGs.addAll(appendedPKGs);
         if(!dryRun&&!debug)
@@ -402,9 +407,56 @@ public class Synchronizer
             //System.out.println();
             //java.....
             PackageState thePackage = packages.get(pak);
-            copylist.addAll(processPackage(packNR, thePackage));
+            copylist.addAll(processPackage(packNR, thePackage, commands));
             packNR++;
         }
+        
+        totalFiles = commands.size();
+        
+        Queue<String[]> commandsToRun = new ConcurrentLinkedQueue<String[]>(commands);
+        l.info("Running convert with {} threads",threads);
+        
+        
+        progressBar = progressBar.withTotalSteps(totalFiles + 1);
+        progressBar.setMessage("Dateien:0/" + totalFiles);
+        progressBar.start();
+        currentFile = 0;
+        Runnable r = ()->
+                {
+                    String[] cmd = commandsToRun.poll();
+                    while(cmd!=null)
+                    {
+                        Tools.runCmdWithPassthru(
+                                IoBuilder.forLogger("External.Imagick").buildPrintStream(),
+                                cmd);
+                        synchronized(progressBar)
+                        {
+                            currentFile++;
+                            progressBar.setMessage("Dateien:" + currentFile + "/" + totalFiles);
+                            progressBar.tickOne();
+                        }
+                        cmd = commandsToRun.poll();
+                    }
+                };
+        Thread[] processingthreads = new Thread[threads];
+        l.info("Spawning {} worker threads",threads);
+        for(int i=0;i<threads;i++)
+        {
+            Thread t = new Thread(r, "WPC-Worker-"+i);
+            processingthreads[i] = t;
+            t.start();
+        }
+        for(int i=0;i<threads;i++)
+        {
+            try{
+                processingthreads[i].join();
+            }
+            catch(InterruptedException e)
+            {
+                
+            };
+        }
+        l.info("Multithreaded processing finished");
         progressBar.complete();
         System.out.println();
         if(dryRun)
@@ -429,6 +481,7 @@ public class Synchronizer
                 currentFile++;
             });
             progressBar.complete();
+            System.out.println();
         }
         else
         {
@@ -438,6 +491,7 @@ public class Synchronizer
         {
             WPCUtils.saveCacheState(cachedir, theState);
         }
+        
         folderOptimize();
         System.out.println();
     }
@@ -536,9 +590,8 @@ public class Synchronizer
         l.trace("Package size of {}={}|{}|{},{}",theState.getPath(),bfs,rfs,ffs,bfs+rfs+ffs);
     }
 
-    private List<ResultWallpaper> processPackage(int packageOffset, PackageState st)
+    private List<ResultWallpaper> processPackage(int packageOffset, PackageState st, List<String[]> commands)
     {
-        l.trace(currentFile); 
         l.trace(new Yaml().dump(st));
         List<String> bframes = new ArrayList<>();
 
@@ -550,22 +603,22 @@ public class Synchronizer
         List<Wallpaper> wpl = st.getB_frames();
         Wallpaper lastSingleWP = null;
         boolean lastPicIsSquashable = true;
-
+        
+        List<String[]> bframecmds = new ArrayList<>();
+        
         int i = 0;
         for (Wallpaper wallpaper : wpl)
         {
             wallpaper.upgrade();
-            progressBar.setMessage("Dateien:" + currentFile + "/" + totalFiles);
             lastSingleWP = wallpaper;
 
             String targetfile = bframedir + "bframe_" + i + ".jpg";
-            processWallpaper(wallpaper, lastSingleWP, targetfile, true, lastPicIsSquashable, true,packageOffset,i);
+            processWallpaper(wallpaper, lastSingleWP, targetfile, true, lastPicIsSquashable, true,packageOffset,i, bframecmds);
             bframes.add(targetfile);
             i++;
-            progressBar.tickOne();
-            currentFile++;
         }
-
+        
+        List<String[]> internalCmds = new ArrayList<>();
         int bframe_count = bframes.size();
         //int bframe_i = 0;
         wpl = st.getStretchablePics();
@@ -573,10 +626,9 @@ public class Synchronizer
         for (Wallpaper wallpaper : wpl)
         {
             wallpaper.upgrade();
-            progressBar.setMessage("Dateien:" + currentFile + "/" + totalFiles);
             String targetFilenameBase = String.format("%04d", packageOffset) + "_a_" + String.format("%06d", i);
             String targetfile = outdir + targetFilenameBase + ".jpg";
-            boolean[] swp = processWallpaper(wallpaper, lastSingleWP, targetfile, true, lastPicIsSquashable, false,packageOffset,i);
+            boolean[] swp = processWallpaper(wallpaper, lastSingleWP, targetfile, true, lastPicIsSquashable, false,packageOffset,i,internalCmds);
             i++;
             if (swp[0])
             {
@@ -592,18 +644,15 @@ public class Synchronizer
                     //bframe_i++;
                 }
             }
-            progressBar.tickOne();
-            currentFile++;
         }
         i = 0;
         wpl = st.getFixedRatioPics();
         for (Wallpaper wallpaper : wpl)
         {
             wallpaper.upgrade();
-            progressBar.setMessage("Dateien:" + currentFile + "/" + totalFiles);
             String targetFilenameBase = String.format("%04d", packageOffset) + "_b_" + String.format("%06d", i);
             String targetfile = outdir + targetFilenameBase + ".jpg";
-            boolean[] swp = processWallpaper(wallpaper, lastSingleWP, targetfile, false, lastPicIsSquashable, false,packageOffset,i);
+            boolean[] swp = processWallpaper(wallpaper, lastSingleWP, targetfile, false, lastPicIsSquashable, false,packageOffset,i,internalCmds);
             i++;
             if (swp[0])
             {
@@ -619,8 +668,13 @@ public class Synchronizer
                     //bframe_i++;
                 }
             }
-            progressBar.tickOne();
-            currentFile++;
+        }
+        
+        //avoid handling bframes if none are required to get copied
+        if(internalCmds.size()>0)
+        {
+            commands.addAll(bframecmds);
+            commands.addAll(internalCmds);
         }
         
         return resultFiles;
@@ -653,6 +707,8 @@ public class Synchronizer
     
     private void identify(Wallpaper wallpaper,String progressBase)
     {
+        l.trace("ID({}/{})",currentFile,totalFiles);
+        
         progressBar.setMessage(progressBase+ currentFile + "/" + totalFiles);
         progressBar.tickOne();
         currentFile++;
@@ -710,10 +766,18 @@ public class Synchronizer
         }
     }
     
-    private boolean[] processWallpaper(Wallpaper wallpaper, Wallpaper fallback, String targetfile, boolean allowStretch, boolean fallbackSquashable, boolean isBFrame,int packageid,int fileoffset)
+    private boolean[] processWallpaper(
+            Wallpaper wallpaper,
+            Wallpaper fallback,
+            String targetfile,
+            boolean allowStretch,
+            boolean fallbackSquashable,
+            boolean isBFrame,
+            int packageid,
+            int fileoffset,
+            List<String[]> commands)
     {
         l.trace(wallpaper.getFilepath());
-        l.trace(currentFile);
         boolean wallpaperIsSingle = false;
         boolean skipped = true;
         if(dryRun)
@@ -792,8 +856,7 @@ public class Synchronizer
                 "-layers", "flatten","-quality",quality, targetfile
             };
             convertCmd.addAll(Arrays.asList(convertEnd));
-
-            Tools.runCmdWithPassthru(IoBuilder.forLogger("External.Imagick").buildPrintStream(), convertCmd.toArray(convertEnd));
+            commands.add(convertCmd.toArray(convertEnd));
         }
         return new boolean[]
         {

izpl-shared/src/main/java/de/nplusc/izc/iZpl/Utils/shared/PLFileIO.java

@@ -224,7 +224,9 @@ public class PLFileIO
                                 try
                                 {
                                     int pr = Integer.valueOf(meta[1]);
-                                    if(pr<1)
+                                    
+                                    // 0 for "disabling" a File so it only links by manual selection
+                                    if(pr<0)
                                     {
                                         throw new NumberFormatException();
                                     }
@@ -233,7 +235,7 @@ public class PLFileIO
                                 catch(NumberFormatException x)
                                 {
                                     syntaxError=true;
-                                    l.error("Syntax error on line {} of file {}: {}",lne,path,"der 2. block bei Include muss eine positive Ganzzahl sein");
+                                    l.error("Syntax error on line {} of file {}: {}",lne,path,"der 2. block bei Include muss eine Ganzzahl >0 sein");
                                     itm.setTargetPlaycount(1);
                                 }
                             }
@@ -250,7 +252,7 @@ public class PLFileIO
                                 try
                                 {
                                     int pr = Integer.valueOf(meta[0]);
-                                    if (pr < 1)
+                                    if (pr < 0)
                                     {
                                         throw new NumberFormatException();
                                     }