1
2 package org.galagosearch.tupleflow.execution;
3
4 import java.io.File;
5 import java.io.IOException;
6 import java.lang.reflect.InvocationTargetException;
7 import java.lang.reflect.Method;
8 import java.lang.reflect.Modifier;
9 import java.util.ArrayList;
10 import java.util.Arrays;
11 import org.galagosearch.tupleflow.Counter;
12 import org.galagosearch.tupleflow.InputClass;
13 import org.galagosearch.tupleflow.OutputClass;
14 import org.galagosearch.tupleflow.Parameters;
15 import org.galagosearch.tupleflow.Processor;
16 import org.galagosearch.tupleflow.TupleFlowParameters;
17 import org.galagosearch.tupleflow.Type;
18 import org.galagosearch.tupleflow.TypeReader;
19
20 /***
21 *
22 * @author trevor
23 */
24 public class Verification {
25 private static class VerificationErrorHandler implements ErrorHandler {
26 FileLocation location;
27 ErrorStore store;
28
29 public VerificationErrorHandler(ErrorStore store, FileLocation location) {
30 this.location = location;
31 this.store = store;
32 }
33
34 public void addWarning(String message) {
35 store.addWarning(location, message);
36 }
37
38 public void addError(String message) {
39 store.addError(location, message);
40 }
41 }
42
43 private static class VerificationParameters implements TupleFlowParameters {
44 Step step;
45 Stage stage;
46
47 public VerificationParameters(Stage stage, Step step) {
48 this.stage = stage;
49 this.step = step;
50 }
51
52 public Counter getCounter(String name) {
53 return null;
54 }
55
56 public Processor getTypeWriter(String specification) throws IOException {
57 return null;
58 }
59
60 public TypeReader getTypeReader(String specification) throws IOException {
61 return null;
62 }
63
64 public boolean writerExists(String specification, String className, String[] order) {
65 StageConnectionPoint point = stage.connections.get(specification);
66
67 if (point == null) {
68 return false;
69 }
70 if (point.type != ConnectionPointType.Output) {
71 return false;
72 }
73 if (!className.equals(point.getClassName())) {
74 return false;
75 }
76 if (!compatibleOrders(order, point.getOrder())) {
77 return false;
78 }
79 return true;
80 }
81
82 public boolean readerExists(String specification, String className, String[] order) {
83 StageConnectionPoint point = stage.connections.get(specification);
84
85 if (point == null) {
86 return false;
87 }
88 if (point.type != ConnectionPointType.Input) {
89 return false;
90 }
91 if (!className.equals(point.getClassName())) {
92 return false;
93 }
94 if (!compatibleOrders(point.getOrder(), order)) {
95 return false;
96 }
97 return true;
98 }
99
100 public Parameters getXML() {
101 return step.getParameters();
102 }
103 }
104
105 /***
106 * Tests to see if two object orders are compatible. By compatible, we mean that
107 * a list of objects in outputOrder is also in inputOrder. This is true if the orders
108 * are identical, but also if inputOrder is more permissive than outputOrder.
109 *
110 * For instance, suppose we are sorting a list of people's names. People typically
111 * have a surname (last name) and a given name (first name). In Galago notation,
112 * consider these two orders you could use:
113 * +surname
114 * +surname +givenName
115 *
116 * If a list is ordered by (+surname +givenName), then it is also ordered by
117 * +surname. The reverse isn't true, though: if you order by +surname, you
118 * haven't necessarily ordered by (+surname +givenName). Therefore:
119 * compatibleOrders({ "+surname" }, { "+surname", "+givenName" }) == false
120 * compatibleOrders({ "+surname", "+givenName" }, { "+surname" }) == true
121 *
122 * @param currentOrder The current order of the data that is supplied.
123 * @param requiredOrder The required order of the data.
124 */
125 public static boolean compatibleOrders(String[] currentOrder, String[] requiredOrder) {
126
127 if (currentOrder.length < requiredOrder.length) {
128 return false;
129 }
130 for (int i = 0; i < requiredOrder.length; i++) {
131 if (!currentOrder[i].equals(requiredOrder[i])) {
132 return false;
133 }
134 }
135
136 return true;
137 }
138
139 public static boolean requireParameters(String[] required, Parameters parameters, ErrorHandler handler) {
140 boolean result = true;
141 for (String key : required) {
142 if (!parameters.containsKey(key)) {
143 handler.addError("The parameter '" + key + "' is required.");
144 result = false;
145 }
146 }
147 return result;
148 }
149
150 public static boolean isOrderAvailable(String typeName, String[] orderSpec) {
151 try {
152 Class typeClass = Class.forName(typeName);
153 Type type = (Type) typeClass.newInstance();
154 return type.getOrder(orderSpec) != null;
155 } catch (Exception e) {
156 return false;
157 }
158 }
159
160 public static boolean isClassAvailable(String name) {
161 try {
162 Class.forName(name);
163 } catch (Exception e) {
164 return false;
165 }
166
167 return true;
168 }
169
170 public static boolean requireOrder(String typeName, String[] orderSpec, ErrorHandler handler) {
171 if (!isOrderAvailable(typeName, orderSpec)) {
172 StringBuilder builder = new StringBuilder();
173
174 for (String orderKey : orderSpec) {
175 builder.append(orderKey);
176 }
177
178 handler.addError(
179 "The order '" + builder.toString() + "' was not found in " + typeName + ".");
180 return false;
181 }
182 return true;
183 }
184
185 public static boolean requireClass(String typeName, ErrorHandler handler) {
186 if (!isClassAvailable(typeName)) {
187 handler.addError("The class '" + typeName + "' could not be found.");
188 return false;
189 }
190 return true;
191 }
192
193 public static boolean requireWriteableFile(String pathname, ErrorHandler handler) {
194 File path = new File(pathname);
195
196 if (path.exists() && !path.isFile()) {
197 handler.addError("Pathname " + pathname + " exists already and isn't a file.");
198 return false;
199 }
200
201 return requireWriteableDirectoryParent(pathname, handler);
202
203 }
204
205 public static boolean requireWriteableDirectory(String pathname, ErrorHandler handler) {
206 File path = new File(pathname);
207
208 if (path.isFile()) {
209 handler.addError("Pathname " + pathname + " is a file, but a directory is required.");
210 return false;
211 }
212
213 if (path.isDirectory() && !path.canWrite()) {
214 handler.addError("Pathname " + pathname + " is a directory, but it isn't writable.");
215 return false;
216 }
217
218 return requireWriteableDirectoryParent(pathname, handler);
219 }
220
221 /***
222 * <p>If pathname exists, returns true. If pathname doesn't exist, checks to
223 * see if it's possible for this process to create something called pathname.</p>
224 *
225 * <p>This method returns false if the closest existing parent directory of pathname
226 * is not writeable (or isn't a directory)</p>
227 *
228 * <p>For example, if filename is /a/b/c/d/e/f, this method will return true if:
229 * <ul>
230 * <li>/a/b/c/d/e/f exists</li>
231 * <li>/a/b/c/d/e/f doesn't exist, but /a/b/c/d/e does, and is writeable</li>
232 * <li>/a/b/d/d/e doesn't exist, but /a/b/c/d does, and is writeable</li>
233 * <li>/a doesn't exist, but / does, and is writeable.</li>
234 * </ul>
235 * </p>
236 */
237 public static boolean requireWriteableDirectoryParent(final String pathname, final ErrorHandler handler) {
238 File path = new File(pathname);
239
240 if (!path.exists()) {
241 String parent = path.getParent();
242
243 while (parent != null && !new File(parent).exists()) {
244 parent = new File(parent).getParent();
245 }
246
247 if (parent == null) {
248 parent = System.getProperty("user.dir");
249 }
250
251 if (!new File(parent).canWrite()) {
252 handler.addError(
253 "Pathname " + pathname + " doesn't exist, and the parent directory isn't writable.");
254 return false;
255 }
256 }
257
258 return true;
259 }
260
261 private static class TypeState {
262 public String className;
263 public String[] order;
264 public boolean defined;
265
266 public TypeState() {
267 this.className = "java.lang.Object";
268 this.order = new String[0];
269 this.defined = false;
270 }
271
272 public TypeState(TypeState state) {
273 this.className = state.getClassName();
274 this.order = state.getOrder();
275 this.defined = state.isDefined();
276 }
277
278 public boolean check(String className, String[] order) {
279 if (!defined) {
280 return true;
281 }
282 return className.equals(this.className) && Verification.compatibleOrders(order,
283 this.order);
284 }
285
286 public String[] getOrder() {
287 return order;
288 }
289
290 public String getClassName() {
291 return className;
292 }
293
294 public void update(String className, String[] order) {
295 this.className = className;
296 this.order = order;
297 this.defined = true;
298 }
299
300 public void setDefined(boolean defined) {
301 this.defined = defined;
302 }
303
304 private boolean isDefined() {
305 return defined;
306 }
307 }
308
309 public static void verify(TypeState state, Stage stage, ArrayList<Step> steps, ErrorStore store) {
310 for (int i = 0; i < steps.size(); i++) {
311 Step step = steps.get(i);
312 boolean isLastStep = (i == (steps.size() - 1));
313
314 if (step instanceof InputStep) {
315
316 InputStep input = (InputStep) step;
317 StageConnectionPoint point = stage.connections.get(input.getId());
318
319 if (point == null) {
320 store.addError(step.getLocation(),
321 "Input references a connection called '" +
322 input.getId() + "', but it isn't listed in the connections section of the stage.");
323 } else {
324 state.update(point.getClassName(), point.getOrder());
325 }
326 } else if (step instanceof OutputStep) {
327
328 OutputStep output = (OutputStep) step;
329 StageConnectionPoint point = stage.connections.get(output.getId());
330
331 if (point == null) {
332 store.addError(step.getLocation(),
333 "Output references a connection called '" +
334 output.getId() + "', but it isn't listed in the connections section of the stage.");
335 } else {
336 if (state.isDefined() && !state.getClassName().equals(point.getClassName())) {
337 store.addError(step.getLocation(), "Previous step makes '" +
338 state.getClassName() + "' objects, but this output connection wants '" +
339 point.getClassName() + "' objects.");
340 } else if (state.isDefined() && !compatibleOrders(state.getOrder(), point.
341 getOrder())) {
342 store.addError(step.getLocation(), "Previous step outputs objects in '" +
343 Arrays.toString(state.getOrder()) + "' order, but incompatible order '" +
344 Arrays.toString(point.getOrder()) + "' is required.");
345 }
346 }
347
348 state.setDefined(false);
349 } else if (step instanceof MultiStep) {
350
351
352 MultiStep multiStep = (MultiStep) step;
353
354 for (ArrayList<Step> group : multiStep.groups) {
355 verify(new TypeState(state), stage, group, store);
356 state.setDefined(false);
357 }
358 } else {
359 Class clazz;
360 try {
361 clazz = Class.forName(step.getClassName());
362 } catch (ClassNotFoundException ex) {
363 store.addError(step.getLocation(), "Couldn't find class: " + step.getClassName());
364 continue;
365 }
366
367 VerificationParameters vp = new VerificationParameters(stage, step);
368
369 verifyInputClass(state, step, clazz, vp, store);
370 verifyStepClass(clazz, step, store, vp);
371
372 if (!isLastStep) {
373 verifyOutputClass(state, clazz, step, store, vp);
374 }
375 }
376 }
377 }
378
379 private static void verifyOutputClass(TypeState state, final Class clazz, final Step step, final ErrorStore store, final VerificationParameters vp) {
380 String[] outputOrder = new String[0];
381 String outputClass = "java.lang.Object";
382
383 try {
384 OutputClass outputClassAnnotation = (OutputClass) clazz.getAnnotation(OutputClass.class);
385
386 if (outputClassAnnotation != null) {
387 outputClass = outputClassAnnotation.className();
388 outputOrder = outputClassAnnotation.order();
389 state.update(outputClass, outputOrder);
390
391 if (!Verification.isClassAvailable(outputClass)) {
392 store.addError(step.getLocation(), step.getClassName() + ": Class " + step.
393 getClassName() + " has an " +
394 "@OutputClass annotation with the class name '" + outputClass +
395 "' which couldn't be found.");
396 state.setDefined(false);
397 } else {
398 state.update(outputClass, outputOrder);
399 }
400 } else {
401 try {
402 Method getOutputClass = clazz.getDeclaredMethod("getOutputClass",
403 TupleFlowParameters.class);
404
405 if (getOutputClass.getReturnType() == String.class) {
406 outputClass = (String) getOutputClass.invoke(null, vp);
407 outputOrder = new String[0];
408
409 try {
410 Method getOutputOrder = clazz.getDeclaredMethod("getOutputOrder",
411 TupleFlowParameters.class);
412 outputOrder = (String[]) getOutputOrder.invoke(null, vp);
413 } catch (NoSuchMethodException e) {
414
415 }
416
417 if (!Verification.isClassAvailable(outputClass)) {
418 store.addError(step.getLocation(),
419 step.getClassName() + ": Class " + step.getClassName() + " returned " +
420 "an output class name '" + outputClass + "' which couldn't be found.");
421 state.setDefined(false);
422 } else {
423 state.update(outputClass, outputOrder);
424 }
425 } else {
426 store.addError(step.getLocation(), step.getClassName() + " has a class method called getOutputClass, " +
427 "but it returns something other than java.lang.String.");
428 state.setDefined(false);
429 }
430 } catch (NoSuchMethodException e) {
431 store.addWarning(step.getLocation(), step.getClassName() + ": Class " + step.
432 getClassName() + " has no suitable " +
433 "getOutputClass method and no @OutputClass annotation.");
434 state.setDefined(false);
435 }
436 }
437 } catch (InvocationTargetException e) {
438 store.addError(step.getLocation(),
439 step.getClassName() + ": Caught an InvocationTargetException while verifying class: " + e.
440 getMessage());
441 state.setDefined(false);
442 } catch (SecurityException e) {
443 store.addError(step.getLocation(),
444 step.getClassName() + ": Caught a SecurityException while verifying class: " + e.
445 getMessage());
446 state.setDefined(false);
447 } catch (IllegalArgumentException e) {
448 store.addError(step.getLocation(),
449 step.getClassName() + ": Caught an IllegalArgumentException while verifying class: " + e.
450 getMessage());
451 state.setDefined(false);
452 } catch (IllegalAccessException e) {
453 store.addError(step.getLocation(),
454 step.getClassName() + ": Caught an IllegalAccessException while verifying class: " + e.
455 getMessage());
456 state.setDefined(false);
457 }
458 }
459
460 private static void verifyStepClass(final Class clazz, final Step step, final ErrorStore store, final VerificationParameters vp) {
461 try {
462 Verified verifiedAnnotation = (Verified) clazz.getAnnotation(Verified.class);
463
464
465 if (verifiedAnnotation != null) {
466 return;
467 }
468 Method verify = clazz.getDeclaredMethod("verify", TupleFlowParameters.class,
469 ErrorHandler.class);
470
471 if (verify == null) {
472 store.addWarning(step.getLocation(), "Class " + step.getClassName() +
473 " has no suitable verify method.");
474 } else if (Modifier.isStatic(verify.getModifiers()) == false) {
475 store.addWarning(step.getLocation(), "Class " + step.getClassName() +
476 " has a verify method, but it isn't static.");
477 } else {
478 verify.invoke(null, vp,
479 new VerificationErrorHandler(store, step.getLocation()));
480 }
481 } catch (InvocationTargetException e) {
482 store.addError(step.getLocation(),
483 step.getClassName() + ": Caught an InvocationTargetException while verifying class: " + e.
484 getMessage());
485 } catch (SecurityException e) {
486 store.addError(step.getLocation(),
487 step.getClassName() + ": Caught a SecurityException while verifying class: " + e.
488 getMessage());
489 } catch (NoSuchMethodException e) {
490 store.addWarning(step.getLocation(),
491 "Class " + step.getClassName() + " has no suitable verify method.");
492 } catch (IllegalArgumentException e) {
493 store.addError(step.getLocation(),
494 step.getClassName() + ": Caught an IllegalArgumentException while verifying class: " + e.
495 getMessage());
496 } catch (IllegalAccessException e) {
497 store.addError(step.getLocation(),
498 step.getClassName() + ": Caught an IllegalAccessException while verifying class: " + e.
499 getMessage());
500 }
501 }
502
503 private static Class findInputClassType(Class clazz) {
504 Method[] allMethods = clazz.getMethods();
505
506 for (Method method : allMethods) {
507 if (!method.getName().equals("process"))
508 continue;
509 Class[] types = method.getParameterTypes();
510 if (types.length != 1)
511 continue;
512 if (types[0] == Object.class)
513 continue;
514 return types[0];
515 }
516 return null;
517 }
518
519 private static void verifyInputClass(TypeState state, final Step step, final Class clazz, final VerificationParameters vp, final ErrorStore store) {
520 if (!state.isDefined()) {
521 return;
522 }
523 try {
524 Class inputClass = findInputClassType(clazz);
525
526 InputClass inputClassAnnotation = (InputClass) clazz.getAnnotation(InputClass.class);
527 String inputClassName = "unknown";
528 String[] inputOrder = new String[0];
529
530 if (inputClassAnnotation != null) {
531 inputClassName = inputClassAnnotation.className();
532 inputOrder = inputClassAnnotation.order();
533
534 if (inputClass != null && !inputClassName.equals(inputClass.getName())) {
535 String outputMessage = String.format("%s: Class %s has an @InputClass " +
536 "annotation with the class name '%s', but the process() method takes " +
537 "'%s' objects.", step.getClassName(), step.getClassName(),
538 inputClassName, inputClass.getName());
539 store.addError(step.getLocation(), outputMessage);
540 }
541
542 if (!Verification.isClassAvailable(inputClassName)) {
543 store.addError(step.getLocation(), step.getClassName() + ": Class " + step.
544 getClassName() + " has an " +
545 "@InputClass annotation with the class name '" + inputClassName +
546 "' which couldn't be found.");
547 }
548 } else if (inputClass != null) {
549 inputClassName = inputClass.getName();
550 } else if (inputClass == null) {
551 try {
552 Method getInputClass = clazz.getDeclaredMethod("getInputClass",
553 TupleFlowParameters.class);
554
555 if (getInputClass.getReturnType() == String.class) {
556 inputClassName = (String) getInputClass.invoke(null, vp);
557
558 try {
559 Method getInputOrder = clazz.getDeclaredMethod("getInputOrder",
560 TupleFlowParameters.class);
561 inputOrder = (String[]) getInputOrder.invoke(null, vp);
562 } catch (NoSuchMethodException e) {
563
564 }
565
566 if (!Verification.isClassAvailable(inputClassName)) {
567 store.addError(step.getLocation(), step.getClassName() + ": Class " + step.getClassName() + " has an " +
568 "returned '" + inputClassName + "' from getInputClass, but " +
569 "it couldn't be found.");
570 }
571 } else {
572 store.addError(step.getLocation(), step.getClassName() + " has a class method called getInputClass, " +
573 "but it returns something other than java.lang.String.");
574 }
575 } catch (NoSuchMethodException e) {
576 store.addWarning(step.getLocation(), step.getClassName() + ": Class " + step.
577 getClassName() + " has no suitable " +
578 "getInputClass method and has no @InputClass annotation.");
579 return;
580 }
581 }
582
583 if (state.isDefined()) {
584 if (!inputClassName.equals(state.getClassName())) {
585 store.addError(step.getLocation(), "Current pipeline class '" + state.
586 getClassName() +
587 "' is different than the required type: '" +
588 inputClassName + "'.");
589 }
590
591 if (!compatibleOrders(state.getOrder(), inputOrder)) {
592 store.addError(step.getLocation(),
593 "Current object order '" + Arrays.toString(state.getOrder()) + "' is incompatible " +
594 "with the required input order: '" + Arrays.toString(inputOrder) + "'.");
595 }
596 }
597 } catch (InvocationTargetException e) {
598 store.addError(step.getLocation(),
599 step.getClassName() + ": Caught an InvocationTargetException while verifying class: " + e.
600 getMessage());
601 } catch (SecurityException e) {
602 store.addError(step.getLocation(),
603 step.getClassName() + ": Caught a SecurityException while verifying class: " + e.
604 getMessage());
605 } catch (IllegalArgumentException e) {
606 store.addError(step.getLocation(),
607 step.getClassName() + ": Caught an IllegalArgumentException while verifying class: " + e.
608 getMessage());
609 } catch (IllegalAccessException e) {
610 store.addError(step.getLocation(),
611 step.getClassName() + ": Caught an IllegalAccessException while verifying class: " + e.
612 getMessage());
613 }
614 }
615
616 public static void verify(Stage stage, ErrorStore store) {
617 TypeState state = new TypeState();
618 verify(state, stage, stage.steps, store);
619 }
620
621 public static void verify(Job job, ErrorStore store) {
622 for (Stage stage : job.stages.values()) {
623 verify(stage, store);
624 }
625 }
626
627 public static boolean verifyTypeReader(String readerName, Class typeClass, TupleFlowParameters parameters, ErrorHandler handler) {
628 return verifyTypeReader(readerName, typeClass, new String[0], parameters, handler);
629 }
630
631 public static boolean verifyTypeReader(String readerName, Class typeClass, String[] order, TupleFlowParameters parameters, ErrorHandler handler) {
632 if (!parameters.readerExists(readerName, typeClass.getName(), order)) {
633 handler.addError("No reader named '" + readerName + "' was found in this stage.");
634 return false;
635 }
636
637 return true;
638 }
639
640 public static boolean verifyTypeWriter(String readerName, Class typeClass, String order[], TupleFlowParameters parameters, ErrorHandler handler) {
641 if (!parameters.writerExists(readerName, typeClass.getName(), order)) {
642 handler.addError("No writer named '" + readerName + "' was found in this stage.");
643 return false;
644 }
645
646 return true;
647 }
648 }