|
|
@@ -15,7 +15,7 @@ import com.eclipsesource.json.*;
|
|
|
|
|
|
public class CSVWiggler {
|
|
|
private static final String EVT_SYS_ID = "958372977714540574";
|
|
|
- public static void main(String[] args) {
|
|
|
+ public static void main(String... args) {
|
|
|
|
|
|
if(args.length<1)
|
|
|
{
|
|
|
@@ -28,143 +28,193 @@ public class CSVWiggler {
|
|
|
{
|
|
|
System.err.println("File not found");
|
|
|
}
|
|
|
+
|
|
|
+ if(checkme.isDirectory())
|
|
|
+ {
|
|
|
+ File[] content = checkme.listFiles();
|
|
|
+ for (File f:content)
|
|
|
+ {
|
|
|
+ if(f.isDirectory())
|
|
|
+ {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ if(f.getName().endsWith(".json"))
|
|
|
+ {
|
|
|
+ main(f.getAbsolutePath());
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
try {
|
|
|
String intermediateFile = args[0] + ".converted.csv";
|
|
|
String cookies = args[0] + ".cookiemetadata.csv";
|
|
|
String mionutely = args[0] + ".minutemetadata.csv";
|
|
|
|
|
|
- JsonObject obj = Json.parse(new FileReader(args[0])).asObject();
|
|
|
- JsonArray arr = obj.get("messages").asArray();
|
|
|
- PrintWriter w = new PrintWriter(new FileWriter(intermediateFile));
|
|
|
- w.println("type;author;length;cookiecount;timestamp");
|
|
|
- for (int i = 0; i < arr.size(); i++) {
|
|
|
- boolean isEventSys = false;
|
|
|
- JsonObject msg = arr.get(i).asObject();
|
|
|
- JsonObject author = msg.get("author").asObject();
|
|
|
- String id = author.getString("id", "---");
|
|
|
- if (id.equals(EVT_SYS_ID)) {
|
|
|
- isEventSys = true;
|
|
|
- System.out.println("Eventsys");
|
|
|
- }
|
|
|
+ convertJsonIntoIntermediateFile(args, intermediateFile);
|
|
|
|
|
|
- String message = msg.getString("content", "");
|
|
|
-
|
|
|
- int len = message.length();
|
|
|
- String timestamp = msg.getString("timestamp", "").substring(0, 19);
|
|
|
- int cookiecount = 0;
|
|
|
- String type = "default";
|
|
|
- if (isEventSys) {
|
|
|
- type = "leaderboard";
|
|
|
- JsonArray embeds = msg.get("embeds").asArray();
|
|
|
- if (!(embeds.size() == 0)) {
|
|
|
- JsonObject embed = embeds.get(0).asObject();
|
|
|
- String title = embed.getString("title", "");
|
|
|
- String content = embed.getString("title", "");
|
|
|
- if (title.equals("Gesammelte Cookies")) {
|
|
|
+
|
|
|
+ processIntermediateFileForCookiesAndMinutely(intermediateFile, cookies, mionutely);
|
|
|
+ }
|
|
|
+ catch (IOException ex)
|
|
|
+ {
|
|
|
+ ex.printStackTrace();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private static void convertJsonIntoIntermediateFile(String[] args, String intermediateFile) throws IOException {
|
|
|
+ JsonObject obj = Json.parse(new FileReader(args[0])).asObject();
|
|
|
+ JsonArray arr = obj.get("messages").asArray();
|
|
|
+ PrintWriter intermediate = new PrintWriter(new FileWriter(intermediateFile));
|
|
|
+ intermediate.println("type;author;length;cookiecount;timestamp");
|
|
|
+ for (int i = 0; i < arr.size(); i++) {
|
|
|
+ boolean isEventSys = false;
|
|
|
+ JsonObject msg = arr.get(i).asObject();
|
|
|
+ JsonObject author = msg.get("author").asObject();
|
|
|
+ String id = author.getString("id", "---");
|
|
|
+ if (id.equals(EVT_SYS_ID)) {
|
|
|
+ isEventSys = true;
|
|
|
+ System.out.println("Eventsys");
|
|
|
+ }
|
|
|
+
|
|
|
+ String message = msg.getString("content", "");
|
|
|
+
|
|
|
+ int len = message.length();
|
|
|
+ String timestamp = msg.getString("timestamp", "").substring(0, 19);
|
|
|
+ int cookiecount = 0;
|
|
|
+ String type = "default";
|
|
|
+ if (isEventSys) {
|
|
|
+ type = "leaderboard";
|
|
|
+ JsonArray embeds = msg.get("embeds").asArray();
|
|
|
+ if (!(embeds.size() == 0)) {
|
|
|
+ JsonObject embed = embeds.get(0).asObject();
|
|
|
+ String title = embed.getString("title", "");
|
|
|
+ String content = embed.getString("title", "");
|
|
|
+ if (title.equals("Gesammelte Cookies")) {
|
|
|
+ type = "cookie";
|
|
|
+ String description = embed.getString("description", "0");
|
|
|
+ String cntS = description.replace("Ihr konntet ", "").replace(" Cookies sammeln.", "").replace(" Cookie sammeln.", "");
|
|
|
+ cookiecount = Integer.valueOf(cntS);
|
|
|
+ }
|
|
|
+ else
|
|
|
+ {
|
|
|
+ if (title.equals("Ein Cookie!"))
|
|
|
+ {
|
|
|
type = "cookie";
|
|
|
- String description = embed.getString("description", "0");
|
|
|
- String cntS = description.replace("Ihr konntet ", "").replace(" Cookies sammeln.", "").replace(" Cookie sammeln.", "");
|
|
|
+ JsonArray fields = embed.get("fields").asArray();
|
|
|
+ JsonObject countField = fields.get(0).asObject();
|
|
|
+ String cntS = countField.getString("value","0");
|
|
|
cookiecount = Integer.valueOf(cntS);
|
|
|
}
|
|
|
- else
|
|
|
- {
|
|
|
- if (title.equals("Ein Cookie!"))
|
|
|
- {
|
|
|
- type = "cookie";
|
|
|
- JsonArray fields = embed.get("fields").asArray();
|
|
|
- JsonObject countField = fields.get(0).asObject();
|
|
|
- String cntS = countField.getString("value","0");
|
|
|
- cookiecount = Integer.valueOf(cntS);
|
|
|
- }
|
|
|
- }
|
|
|
}
|
|
|
}
|
|
|
- w.println(type + ";" + id + ";" + len + ";" + cookiecount + ";" + timestamp);
|
|
|
}
|
|
|
- w.close();
|
|
|
-
|
|
|
- w = new PrintWriter(new FileWriter(cookies));
|
|
|
- PrintWriter w2 = new PrintWriter(new FileWriter(mionutely));
|
|
|
-
|
|
|
- BufferedReader r = new BufferedReader(new FileReader(intermediateFile));
|
|
|
- int lns = -100000;
|
|
|
- int lnsMinutely = -100000;
|
|
|
- String line = r.readLine();
|
|
|
- line = r.readLine(); //datei köpfen
|
|
|
- HashSet<String> users = new HashSet<>();
|
|
|
- HashSet<String> usersMinutely = new HashSet<>();
|
|
|
- String previousLineTS = "";
|
|
|
-
|
|
|
- w.println("cookiedelta;usercount;type;author;length;cookiecount;timestamp");
|
|
|
- w2.println("usercount;msgsMinute;timestamp");
|
|
|
- while (line != null) {
|
|
|
- String[] splitted = line.split(";");
|
|
|
- if (line.contains("cookie;")) {
|
|
|
- String lnp = lns+"";
|
|
|
- if (lns < 0) {
|
|
|
- lnp = "-";
|
|
|
- }
|
|
|
- System.out.println(lns+"|"+lnp);
|
|
|
- w.println(lnp+";" +users.size()+";"+ line);
|
|
|
- lns = 0;
|
|
|
- users=new HashSet<>();
|
|
|
- } else {
|
|
|
- String uid = splitted[1];
|
|
|
- //if(!uid.equals(EVT_SYS_ID))
|
|
|
- //{
|
|
|
- lns++;
|
|
|
- users.add(uid);
|
|
|
- //}
|
|
|
+ intermediate.println(type + ";" + id + ";" + len + ";" + cookiecount + ";" + timestamp);
|
|
|
+ }
|
|
|
+ intermediate.close();
|
|
|
+ }
|
|
|
+
|
|
|
+ private static void processIntermediateFileForCookiesAndMinutely(String intermediateFile, String cookies, String mionutely) throws IOException {
|
|
|
+ PrintWriter cookielog = new PrintWriter(new FileWriter(cookies));
|
|
|
+ PrintWriter minutelog = new PrintWriter(new FileWriter(mionutely));
|
|
|
+
|
|
|
+ BufferedReader r = new BufferedReader(new FileReader(intermediateFile));
|
|
|
+ int lns = -100000;
|
|
|
+ int lnsMinutely = -100000;
|
|
|
+ int chars = -100000;
|
|
|
+ String line = r.readLine();
|
|
|
+ line = r.readLine(); //datei köpfen
|
|
|
+ HashSet<String> users = new HashSet<>();
|
|
|
+ HashSet<String> usersMinutely = new HashSet<>();
|
|
|
+
|
|
|
+ String previousLineTS = "";
|
|
|
+
|
|
|
+ boolean keks=false;
|
|
|
+ String keksC = "";
|
|
|
+ cookielog.println("cookiedelta;usercount;type;author;length;cookiecount;timestamp");
|
|
|
+ minutelog.println("usercount;msgsMinute;timestamp;usersAccumulated;messagesAccumulated;cookie;cookiecount");
|
|
|
+ while (line != null) {
|
|
|
+ String[] splitted = line.split(";");
|
|
|
+ if (line.contains("cookie;")) {
|
|
|
+ String lnp = lns+"";
|
|
|
+ String chp = chars+"";
|
|
|
+ if (lns < 0) {
|
|
|
+ lnp = "-";
|
|
|
+ }
|
|
|
+ if (chars < 0) {
|
|
|
+ chp = "-";
|
|
|
}
|
|
|
+ keksC = splitted[3];
|
|
|
+ System.out.println(lns+"|"+lnp);
|
|
|
+ cookielog.println(lnp+";" +users.size()+";"+ line);
|
|
|
+ lns = 0;
|
|
|
+ users=new HashSet<>();
|
|
|
+ keks=true;
|
|
|
+ chars=0;
|
|
|
+ } else {
|
|
|
+ String uid = splitted[1];
|
|
|
+ //if(!uid.equals(EVT_SYS_ID))
|
|
|
+ //{
|
|
|
+ lns++;
|
|
|
+ users.add(uid);
|
|
|
+ chars+=Integer.valueOf(splitted[2]);
|
|
|
+ //}
|
|
|
+ }
|
|
|
+
|
|
|
+ String tsMinutely = splitted[4].substring(0, 16);
|
|
|
+ if(!previousLineTS.equals(tsMinutely))
|
|
|
+ {
|
|
|
+ SimpleDateFormat parser = new SimpleDateFormat("yyyy-MM-dd_HH:mm");
|
|
|
|
|
|
- String tsMinutely = splitted[4].substring(0, 16);
|
|
|
- if(!previousLineTS.equals(tsMinutely))
|
|
|
+ int lnsInternal = lns;
|
|
|
+ if(lns<0)
|
|
|
{
|
|
|
- SimpleDateFormat parser = new SimpleDateFormat("yyyy-MM-dd_HH:mm");
|
|
|
- if(previousLineTS.length()>0)//suppress calc on head of file
|
|
|
- {
|
|
|
- int minutesToSynthesize = 0;
|
|
|
- try {
|
|
|
- Date old = parser.parse(previousLineTS.replace("T","_"));
|
|
|
- Instant calc= old.toInstant();
|
|
|
- Date current = parser.parse(tsMinutely.replace("T","_"));
|
|
|
- Long gap = ChronoUnit.MINUTES.between(old.toInstant(),current.toInstant());
|
|
|
- if(gap>1)
|
|
|
+ lnsInternal=0;
|
|
|
+ }
|
|
|
+ String linetail = ";"+users.size()+";"+lnsInternal+";"+(keks?"100":"0")+";"+keksC;
|
|
|
+
|
|
|
+ if(previousLineTS.length()>0)//suppress calc on head of file
|
|
|
+ {
|
|
|
+ int minutesToSynthesize = 0;
|
|
|
+ try {
|
|
|
+ Date old = parser.parse(previousLineTS.replace("T","_"));
|
|
|
+ Instant calc= old.toInstant();
|
|
|
+ Date current = parser.parse(tsMinutely.replace("T","_"));
|
|
|
+ Long gap = ChronoUnit.MINUTES.between(old.toInstant(),current.toInstant());
|
|
|
+ if(gap>1)
|
|
|
+ {
|
|
|
+ System.out.println("Gap:"+gap+"||"+previousLineTS+"|"+tsMinutely);
|
|
|
+ System.out.println(">>"+old+"|"+current);
|
|
|
+ for(int g=1;g<gap;g++)
|
|
|
{
|
|
|
- System.out.println("Gap:"+gap+"||"+previousLineTS+"|"+tsMinutely);
|
|
|
- System.out.println(">>"+old+"|"+current);
|
|
|
- for(int g=1;g<gap;g++)
|
|
|
- {
|
|
|
- String ts = parser.format(Date.from(calc.plus(g, ChronoUnit.MINUTES)));
|
|
|
- w2.println("0;0;"+(ts.replace("_","T")));
|
|
|
- }
|
|
|
+ String ts = parser.format(Date.from(calc.plus(g, ChronoUnit.MINUTES)));
|
|
|
+ minutelog.println("0;0;"+(ts.replace("_","T"))+linetail);
|
|
|
}
|
|
|
- } catch (ParseException e) {
|
|
|
- throw new RuntimeException(e);
|
|
|
}
|
|
|
+ } catch (ParseException e) {
|
|
|
+ throw new RuntimeException(e);
|
|
|
}
|
|
|
- previousLineTS=tsMinutely;
|
|
|
- if(lnsMinutely>=0)
|
|
|
- {
|
|
|
- w2.println(+usersMinutely.size()+";"+lnsMinutely+";"+previousLineTS);
|
|
|
- }
|
|
|
- lnsMinutely=0;
|
|
|
- usersMinutely=new HashSet<>();
|
|
|
}
|
|
|
- String uid = splitted[1];
|
|
|
- if(!uid.equals(EVT_SYS_ID))
|
|
|
+ previousLineTS=tsMinutely;
|
|
|
+ if(lnsMinutely>=0)
|
|
|
{
|
|
|
- lnsMinutely++;
|
|
|
- usersMinutely.add(uid);
|
|
|
+ minutelog.println(+usersMinutely.size()+";"+lnsMinutely+";"+previousLineTS+linetail);
|
|
|
}
|
|
|
- line = r.readLine();
|
|
|
+ lnsMinutely=0;
|
|
|
+ usersMinutely=new HashSet<>();
|
|
|
+ keks=false;
|
|
|
+ keksC="";
|
|
|
}
|
|
|
- w.close();
|
|
|
- w2.close();
|
|
|
- }
|
|
|
- catch (IOException ex)
|
|
|
- {
|
|
|
- ex.printStackTrace();
|
|
|
+ String uid = splitted[1];
|
|
|
+ if(!uid.equals(EVT_SYS_ID))
|
|
|
+ {
|
|
|
+ lnsMinutely++;
|
|
|
+ usersMinutely.add(uid);
|
|
|
+ }
|
|
|
+ line = r.readLine();
|
|
|
}
|
|
|
+ cookielog.close();
|
|
|
+ minutelog.close();
|
|
|
}
|
|
|
}
|
