1 package org.kit.furia.fragment;
2
3 import java.io.BufferedInputStream;
4 import java.io.BufferedReader;
5 import java.io.File;
6 import java.io.FileNotFoundException;
7 import java.io.FileWriter;
8 import java.io.IOException;
9 import java.io.InputStream;
10 import java.io.InputStreamReader;
11 import java.util.Arrays;
12 import java.util.LinkedList;
13 import java.util.List;
14 import java.util.Map;
15 import java.util.concurrent.CountDownLatch;
16 import java.util.concurrent.atomic.AtomicInteger;
17
18 import org.ajmm.obsearch.asserts.OBAsserts;
19 import org.apache.log4j.Logger;
20 import org.apache.log4j.PropertyConfigurator;
21 import org.kit.furia.misc.FuriaProperties;
22
23 /*
24 Furia-chan: An Open Source software license violation detector.
25 Copyright (C) 2007 Kyushu Institute of Technology
26
27 This program is free software: you can redistribute it and/or modify
28 it under the terms of the GNU General Public License as published by
29 the Free Software Foundation, either version 3 of the License, or
30 (at your option) any later version.
31
32 This program is distributed in the hope that it will be useful,
33 but WITHOUT ANY WARRANTY; without even the implied warranty of
34 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
35 GNU General Public License for more details.
36
37 You should have received a copy of the GNU General Public License
38 along with this program. If not, see <http://www.gnu.org/licenses/>.
39 */
40
41 /**
42 * FragmentBuilderClient is in charge of executing soot in a set of java class
43 * directories and leaving a "fragments" file in the specified output directory.
44 * Since Soot has problems when executed several times during the life of a VM
45 * (reset had issues) we call a new VM for each program that will be fragmented.
46 * @author Arnoldo Jose Muller Molina
47 * @since 0
48 */
49
50 public class FragmentBuilderClient {
51
52 /**
53 * Used to stop the threads if an error occurs.
54 */
55 private Exception exception = null;
56
57 /**
58 * Logger.
59 */
60 private static Logger logger = Logger.getLogger("FragmentBuilderClient");
61
62 /**
63 * Files that will be processed.
64 */
65 private File[] filesToProcess;
66
67 /**
68 * Index used to access filesToProcess.
69 */
70 private AtomicInteger i;
71
72 /**
73 * Used to wait until all threads have finished.
74 */
75 private CountDownLatch join;
76
77 /**
78 * Maximum # of expansions performed for each fragment.
79 */
80 private int maxExpansions;
81
82 /**
83 * If the program will receive one directory (that holds in its root a set
84 * of java class files) (false) or if the directory received contains
85 * directories that contain class files. Each of these directories is
86 * treated as a different application.
87 */
88 private boolean directoryOfDirectoriesMode;
89
90 /**
91 * The output directory to use.
92 */
93 protected File outputDirectory;
94
95 /**
96 * Fail if there is an error.
97 */
98 private boolean failOnError;
99
100 /**
101 * Number of cpus.
102 */
103 private int cpus;
104
105 /**
106 * Timeout for subprocesses.
107 */
108 private long timeout;
109
110 /**
111 * The frequency we will check each process to complete.
112 */
113 private static final long POLL_INTERVAL = 1000 * 5; // every seconds.
114
115 /**
116 * Error code returned by the caller if there is a timeout exception.
117 */
118 private static final int TIMEOUT_ERRORCODE = -20; // every seconds.
119
120 /**
121 * Fragment engine to employ.
122 */
123 private String engine;
124
125 /**
126 * Takes a directory of a set of directories and generates fragments out of
127 * the given folders. An infinite timeout is set. (Not recommended for Soot)
128 * @param cpus
129 * The number of CPUS to employ
130 * @param directory
131 * The directory that will be opened.
132 * @param directoryOfDirectoriesMode
133 * If the program will receive one directory (that holds in
134 * its root a set of java class files) (false) or if the
135 * directory received contains directories that contain class
136 * files. Each of these directories is treated as a different
137 * application.
138 * @param outputDirectory
139 * The directory that holds the resulting files from the
140 * operation. If directoryOfDirectoriesMode == false then the
141 * output data files will be copied directory to
142 * outputDirectory. Otherwise a directory outputDirectory/<app>
143 * will be created for each application where <app> is the
144 * application name.
145 * @param engine Fragment engine that will be used 'soot' or 'asm'.
146 * @param failOnError
147 * If true, stops if there is an error.
148 */
149 public FragmentBuilderClient(boolean directoryOfDirectoriesMode,
150 File directory, int cpus, File outputDirectory, boolean failOnError, String engine)
151 throws Exception, InterruptedException {
152 this(directoryOfDirectoriesMode, directory, cpus, outputDirectory,
153 failOnError, (long) 0, engine);
154 }
155
156 /**
157 * Takes a directory of a set of directories and generates fragments out of
158 * the given folders.
159 * @param cpus
160 * The number of CPUS to employ
161 * @param directory
162 * The directory that will be opened.
163 * @param directoryOfDirectoriesMode
164 * If the program will receive one directory (that holds in
165 * its root a set of java class files) (false) or if the
166 * directory received contains directories that contain class
167 * files. Each of these directories is treated as a different
168 * application.
169 * @param outputDirectory
170 * The directory that holds the resulting files from the
171 * operation. If directoryOfDirectoriesMode == false then the
172 * output data files will be copied directory to
173 * outputDirectory. Otherwise a directory outputDirectory/<app>
174 * will be created for each application where <app> is the
175 * application name.
176 * @param failOnError
177 * If true, stops if there is an error.
178 * @param fragmentEngine The fragment engine to be used, currently soot or ASM.
179 * @param timeout
180 * Amount of time in milliseconds to wait until one program
181 * is fragmented.
182 */
183 public FragmentBuilderClient(boolean directoryOfDirectoriesMode,
184 File directory, int cpus, File outputDirectory,
185 boolean failOnError, long timeout, String fragmentEngine) throws Exception,
186 InterruptedException {
187 if(! ( fragmentEngine.equals("asm") || fragmentEngine.equals("soot"))){
188 throw new IllegalArgumentException("Engines available are 'asm' or 'soot'.");
189 }
190 this.engine = fragmentEngine;
191 this.timeout = timeout;
192 this.cpus = cpus;
193
194 if (directoryOfDirectoriesMode) {
195 filesToProcess = directory.listFiles();
196 } else {
197 filesToProcess = new File[1];
198 filesToProcess[0] = directory;
199 }
200 this.directoryOfDirectoriesMode = directoryOfDirectoriesMode;
201 this.outputDirectory = outputDirectory;
202 this.i = new AtomicInteger(0);
203 join = new CountDownLatch(filesToProcess.length);
204 this.failOnError = failOnError;
205 int i = 0;
206 while (i < cpus) {
207 new Thread(new FragmentExecutor()).start();
208 i++;
209 }
210 boolean interrupted = true;
211 // wait for all the threads to complete
212 while (interrupted) {
213 try {
214 join.await();
215 interrupted = false;
216 } catch (InterruptedException e) {
217 // dancing all alone...
218 }
219 }
220
221 if (exception != null) {
222 throw exception;
223 }
224
225 }
226
227 /**
228 * Executes a new jvm on each folder and creates the fragments
229 * @author Arnoldo Jose Muller Molina
230 */
231 private class FragmentExecutor implements Runnable {
232
233 public void run() {
234
235 try {
236
237 while (exception == null) {
238 int cx = i.getAndIncrement();
239 if (cx < filesToProcess.length) {
240 File dirToProcess = filesToProcess[cx];
241 logger.info("Processing: " + (cx + 1) + " of "
242 + filesToProcess.length + " % " + dirToProcess
243 + " count: " + join.getCount());
244 execApp(dirToProcess);
245 join.countDown();
246 // if there is an error in the execution, we expect
247 } else {
248 break;
249 }
250 }
251 if (exception != null) {
252 logger
253 .fatal("Quitting thread because exception was not null "
254 + exception.toString());
255 while (join.getCount() > 0) {
256 join.countDown();
257 }
258 }
259
260 logger.debug("Quitting thread");
261
262 } catch (Exception e) {
263 logger.fatal("Caught Exception", e);
264 }
265
266 }
267
268 /**
269 * Calls a sub-process that will fragment
270 * @param dirToProcess
271 */
272 private void execApp(File dirToProcess) {
273 File appOutputDir;
274 // fixed the proper output directory
275 if (directoryOfDirectoriesMode) {
276 appOutputDir = new File(outputDirectory, dirToProcess.getName());
277 } else {
278 appOutputDir = outputDirectory;
279 }
280
281 List < String > command = new LinkedList < String >();
282 command.add("java");
283 long ramToAllocate = Runtime.getRuntime().maxMemory() / 1024 / 1024
284 ;
285 command.add("-Xmx" + (ramToAllocate) + "m");
286 command.add(FragmentBuilderClientAux.class.getCanonicalName());
287 command.add(dirToProcess.toString());
288 command.add(appOutputDir.toString());
289 command.add(engine);
290
291 try {
292 // create dir if it is not created
293 if (!outputDirectory.exists()) {
294 outputDirectory.mkdirs();
295 }
296 OBAsserts.chkAssert(outputDirectory.exists(), "Directory "
297 + outputDirectory + " was not created succesfully");
298
299 // add the current directory as part of the classpath.
300 ProcessBuilder pb = new ProcessBuilder(command);
301 pb.directory(new File(System.getProperty("user.dir")));
302 Map < String, String > env = pb.environment();
303
304 env.put("CLASSPATH", System.getProperty("java.class.path"));
305 env
306 .put("log4j.file", FuriaProperties
307 .getProperty("log4j.file"));
308 pb.redirectErrorStream(true);
309 long endTime = System.currentTimeMillis() + timeout;
310 Process p = pb.start();
311 boolean interrupted = true;
312 // while (interrupted) {
313 StringBuilder x = null;
314 try {
315 InputStream in = p.getInputStream();
316 InputStreamReader inR = new InputStreamReader(in);
317 BufferedReader bIn = new BufferedReader(inR);
318
319 int res = TIMEOUT_ERRORCODE; // code for timeout
320 if (timeout == 0) {
321 res = p.waitFor();
322 } else {
323 boolean finished = false;
324 Object lock = new Object();
325 while (!finished) {
326 try {
327 res = p.exitValue();
328 finished = true;
329 } catch (IllegalThreadStateException notFinished) {
330 try {
331 synchronized (lock) {
332 lock.wait(POLL_INTERVAL);
333 }
334 } catch (InterruptedException waitInterrupted) {
335 // nothing to do.
336 }
337 if (endTime < System.currentTimeMillis()) {
338 // we have to stop
339 finished = true;
340 res = TIMEOUT_ERRORCODE; // :)
341 }
342 }
343 }
344 }
345
346 logger.info("Completed app:" + dirToProcess.toString()
347 + " count " + join.getCount());
348
349 if (res != 0) {
350
351 // Read the output of the invoked program and
352 // print it to string.
353 String line = bIn.readLine();
354 x = new StringBuilder();
355 while (line != null) {
356 x.append(line + "\n");
357 line = bIn.readLine();
358 }
359 bIn.close();
360 FileWriter result = new FileWriter(new File(
361 appOutputDir, "fatal.txt"));
362 // add the msg timeout to the result.
363 if (res == TIMEOUT_ERRORCODE) {
364 x.append("\n Timeout ! :(\n ");
365 }
366 result.write(x.toString());
367 result.close();
368 if (failOnError) {
369 throw new Exception(
370 "Failed while executing command:"
371 + command.toString()
372 + " returned code: " + res + "\n"
373 + x.toString());
374 } else {
375 if(res == 7){
376 logger.warn("No clases found for: "
377 + dirToProcess
378 );
379 }else{
380 logger.warn("Failed to process: "
381 + dirToProcess
382 + " check "
383 + (new File(appOutputDir, "fatal.txt"))
384 .toString());
385 }
386 }
387 }
388 interrupted = false;
389 } catch (InterruptedException e) {
390 logger.fatal("Process interrupted");
391 } catch (FileNotFoundException e2) { // we could not even
392 // create the fatal.txt
393 // msg.
394 if (x != null) {
395 throw new Exception("Failed while executing command:"
396 + command.toString() + "\n" + "\n"
397 + x.toString());// + "\n Passed env:\n" +
398 // pb.environment().toString()
399 // + " \nworking dir: " +
400 // pb.directory());
401 }
402 }
403 // }
404
405 } catch (Exception e) {
406 logger.fatal("Error while executing command", e);
407 exception = e;
408 }
409 }
410
411 }
412
413 }