From 16661918c849c6d07268ff19fb51e6bb3cde2708 Mon Sep 17 00:00:00 2001 From: Manos Koukoutos <emmanouil.koukoutos@epfl.ch> Date: Tue, 8 Mar 2016 17:34:31 +0100 Subject: [PATCH] Synthesis benchmarking --- synthesis-collective.txt | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 synthesis-collective.txt diff --git a/synthesis-collective.txt b/synthesis-collective.txt new file mode 100644 index 000000000..acf809699 --- /dev/null +++ b/synthesis-collective.txt @@ -0,0 +1,33 @@ +Versions: X-X-N for CEGIS(o/n), grammar(o/n), max. size +✓ = Proof +? = no proof +x = invalid solution +F = Timeout/failed to synth. + + +==================================================================================================================================================================== +Name | FDefs| Size | O-O-5 | O-O-7 | 6cf54fc | e710631 | 1d71892 | 1588205 | 7c6ca79 | 904b6ce | 79fd5c5 | 93fda91 | | +==================================================================================================================================================================== +List.insert | 59 | 3 | ✓ | 0.8 | ✓ | 1.0 | ✓ | 0.8 | ✓ | 0.8 | ✓ | 0.8 | ✓ | 0.8 | ✓ | 0.8 | ✓ | 0.9 | ✓ | 0.8 | ✓ | 0.6 | ✓ | 0.7 | +List.delete | 61 | 0 | ✓ | 4.5 | F | 30.1 | ✓ | 9.8 | ✓ | 4.3 | ✓ | 4.3 | ✓ | 4.2 | ✓ | 4.3 | ✓ | 4.0 | ✓ | 3.5 | ✓ | 3.6 | ✓ | 3.2 | +List.union | 75 | 11 | ✓ | 7.1 | ✓ | 12.5 | ✓ | 3.8 | ✓ | 4.2 | ✓ | 4.3 | ✓ | 4.3 | ✓ | 4.5 | ✓ | 3.8 | ✓ | 3.5 | ✓ | 3.5 | ✓ | 3.5 | +List.diff | 106 | 0 | ✓ | 6.1 | F | 31.8 | ✓ | 13.1 | ✓ | 12.6 | ✓ | 13.2 | ✓ | 12.0 | ✓ | 11.3 | ✓ | 11.8 | ✓ | 12.0 | ✓ | 11.6 | ✓ | 11.2 | +List.split*(easier for O-O) | 96 | 24 | ✓ | 3.6 | ✓ | 5.3 | ✓ | 20.0 | ✓ | 17.5 | ✓ | 18.7 | ✓ | 18.8 | ✓ | 17.0 | ✓ | 17.6 | ✓ | 2.0 | ✓ | 2.0 | ✓ | 2.6 | +SortedList.insert | 91 | 0 | ? | 16.5 | F | 30.0 | ? | 39.9 | ? | 23.3 | ? | 22.3 | ? | 24.3 | ? | 23.1 | ? | 22.4 | ? | 22.7 | ? | 20.7 | ? | 23.6 | +SortedList.insertAlways | 105 | 0 | ✓ | 20.5 | F | 30.0 | ✓ | 44.5 | ✓ | 37.4 | ✓ | 36.1 | ✓ | 37.8 | ✓ | 36.0 | ✓ | 38.6 | ✓ | 40.7 | ✓ | 34.3 | ✓ | 40.8 | +SortedList.delete | 91 | 0 | ? | 7.9 | F | 30.0 | ? | 10.8 | ? | 6.8 | ? | 6.7 | ? | 6.7 | ? | 6.6 | ? | 5.9 | ? | 5.9 | ? | 5.6 | ? | 5.7 | +SortedList.union | 138 | 11 | ✓ | 6.9 | ✓ | 15.1 | ✓ | 4.2 | ✓ | 4.4 | ✓ | 4.6 | ✓ | 4.8 | ✓ | 4.5 | ✓ | 3.7 | ✓ | 3.7 | ✓ | 3.8 | ✓ | 4.0 | +SortedList.diff | 136 | 0 | ✓ | 5.9 | F | 51.7 | ✓ | 8.3 | ✓ | 8.4 | ✓ | 8.2 | ✓ | 9.1 | ✓ | 7.2 | ✓ | 6.7 | ✓ | 6.7 | ✓ | 6.6 | ✓ | 7.7 | +SortedList.insertionSort | 125 | 10 | ✓ | 1.9 | ✓ | 3.5 | ✓ | 2.4 | ✓ | 2.2 | ✓ | 2.3 | ✓ | 2.2 | ✓ | 2.1 | ✓ | 1.9 | ✓ | 1.5 | ✓ | 3.3 | ✓ | 1.7 | +StrictSortedList.insert | 91 | 0 | ✓ | 13.1 | F | 30.0 | ✓ | 45.0 | ✓ | 25.9 | ✓ | 25.7 | ✓ | 22.0 | ✓ | 25.5 | ✓ | 22.4 | ✓ | 23.0 | ✓ | 21.3 | ✓ | 23.4 | +StrictSortedList.delete | 91 | 0 | ? | 10.2 | F | 30.0 | ? | 15.0 | ? | 8.9 | ? | 8.8 | ? | 9.0 | ? | 8.6 | ? | 7.8 | ? | 5.6 | ✓ | 5.3 | ✓ | 5.6 | +StrictSortedList.union | 138 | 11 | ✓ | 7.6 | ✓ | 13.6 | ✓ | 4.5 | ✓ | 4.8 | ✓ | 5.0 | ✓ | 4.8 | ✓ | 4.6 | ✓ | 4.0 | ✓ | 3.5 | ✓ | 3.6 | ✓ | 4.7 | +UnaryNumerals.add | 42 | 9 | ✓ | 4.3 | ✓ | 4.1 | ✓ | 2.7 | ✓ | 2.7 | ✓ | 2.6 | ✓ | 2.6 | ✓ | 2.5 | ✓ | 2.0 | ✓ | 2.1 | ✓ | 2.2 | ✓ | 2.2 | +UnaryNumerals.distinct | 66 | 4 | ✓ | 2.4 | ✓ | 2.3 | ✓ | 2.3 | ✓ | 2.2 | ✓ | 2.2 | ✓ | 2.3 | ✓ | 2.4 | ✓ | 2.0 | ✓ | 1.6 | ✓ | 1.1 | ✓ | 1.1 | +UnaryNumerals.mult | 64 | 10 | ✓ | 4.5 | ✓ | 10.5 | ✓ | 7.1 | ✓ | 5.0 | ✓ | 5.0 | ✓ | 5.1 | ✓ | 4.5 | ✓ | 4.1 | ✓ | 4.2 | ✓ | 4.2 | ✓ | 4.4 | +BatchedQueue.enqueue | 89 | 20 | F | 30.0 | ? | 19.7 | ? | 9.4 | ? | 14.2 | ? | 13.3 | ? | 14.1 | ? | 12.9 | ? | 11.0 | ? | 11.9 | ? | 9.9 | ? | 10.0 | +BatchedQueue.dequeue | 65 | 9 | ? | 14.3 | ? | 11.4 | ? | 8.4 | ? | 7.6 | ? | 8.3 | ? | 8.2 | ? | 7.3 | ? | 6.8 | ? | 7.6 | ? | 12.9 | ? | 13.3 | +AddressBook.makeAddressBook | 42 | 0 | F | 14.3 | F | 30.0 | ✓ | 7.5 | ✓ | 25.2 | ✓ | 6.9 | ✓ | 7.2 | ✓ | 6.7 | ✓ | 5.6 | ✓ | 6.3 | ✓ | 6.6 | ✓ | 6.7 | +AddressBook.merge | 99 | 11 | ? | 6.9 | ? | 9.5 | ? | 7.6 | ? | 8.1 | ? | 6.1 | ? | 6.3 | ? | 6.1 | ? | 5.6 | ? | 5.5 | ? | 9.6 | ? | 9.9 | +RunLength.encode | 110 | 38 | | | | | | | | | | | | | | | | | | | ✓ | 50.9 | ✓ | 54.0 | +==================================================================================================================================================================== -- GitLab