Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision
  • final-solutions
  • master
  • sguillou-master-patch-05600
  • sguillou-master-patch-48280
  • sguillou-master-patch-52410
5 results

Target

Select target project
  • lamp/cs206
  • bwermeil/cs206-2020
  • zabifade/cs206-2020
  • cauderan/cs206-2020
  • malonga/cs206-2020
  • dumoncel/cs206
  • bounekhe/cs206
  • bergerault/cs206
  • flealsan/cs206
  • hsu/cs206
  • mouchel/cs206
  • vebraun/cs206
  • vcanard/cs206
  • ybelghmi/cs206
  • belghmi/cs206
  • bousbina/cs206
  • waked/cs206
  • gtagemou/cs206
  • arahmoun/cs206
  • elhachem/cs206
  • benrahha/cs206
  • benslima/cs206
22 results
Select Git revision
  • bergerault-master-patch-06528
  • master
2 results
Show changes
Hide whitespace changes
Inline Side-by-side
Showing
with 0 additions and 41093 deletions
previous-exams/2021-final-solutions/f3/src/test/scala/f3/instrumentation/AtomicReference.scala deleted 100644 → 0
View file @ 071256c3
package f3.instrumentation
class AtomicReference[T](initial: T) {
private val atomic = new java.util.concurrent.atomic.AtomicReference[T](initial)
def get: T = atomic.get()
def set(value: T): Unit = atomic.set(value)
}
previous-exams/2021-final-solutions/f3/src/test/scala/f3/instrumentation/MockedMonitor.scala deleted 100644 → 0
View file @ 071256c3
package f3.instrumentation
trait MockedMonitor extends Monitor {
def scheduler: Scheduler
// Can be overriden.
override def waitDefault() = {
scheduler.log("wait")
scheduler updateThreadState Wait(this, scheduler.threadLocks.tail)
}
override def synchronizedDefault[T](toExecute: =>T): T = {
scheduler.log("synchronized check")
val prevLocks = scheduler.threadLocks
scheduler updateThreadState Sync(this, prevLocks) // If this belongs to prevLocks, should just continue.
scheduler.log("synchronized -> enter")
try {
toExecute
} finally {
scheduler updateThreadState Running(prevLocks)
scheduler.log("synchronized -> out")
}
}
override def notifyDefault() = {
scheduler mapOtherStates {
state => state match {
case Wait(lockToAquire, locks) if lockToAquire == this => SyncUnique(this, state.locks)
case e => e
}
}
scheduler.log("notify")
}
override def notifyAllDefault() = {
scheduler mapOtherStates {
state => state match {
case Wait(lockToAquire, locks) if lockToAquire == this => Sync(this, state.locks)
case SyncUnique(lockToAquire, locks) if lockToAquire == this => Sync(this, state.locks)
case e => e
}
}
scheduler.log("notifyAll")
}
}
trait LockFreeMonitor extends Monitor {
override def waitDefault() = {
throw new Exception("Please use lock-free structures and do not use wait()")
}
override def synchronizedDefault[T](toExecute: =>T): T = {
throw new Exception("Please use lock-free structures and do not use synchronized()")
}
override def notifyDefault() = {
throw new Exception("Please use lock-free structures and do not use notify()")
}
override def notifyAllDefault() = {
throw new Exception("Please use lock-free structures and do not use notifyAll()")
}
}
abstract class ThreadState {
def locks: Seq[AnyRef]
}
trait CanContinueIfAcquiresLock extends ThreadState {
def lockToAquire: AnyRef
}
case object Start extends ThreadState { def locks: Seq[AnyRef] = Seq.empty }
case object End extends ThreadState { def locks: Seq[AnyRef] = Seq.empty }
case class Wait(lockToAquire: AnyRef, locks: Seq[AnyRef]) extends ThreadState
case class SyncUnique(lockToAquire: AnyRef, locks: Seq[AnyRef]) extends ThreadState with CanContinueIfAcquiresLock
case class Sync(lockToAquire: AnyRef, locks: Seq[AnyRef]) extends ThreadState with CanContinueIfAcquiresLock
case class Running(locks: Seq[AnyRef]) extends ThreadState
case class VariableReadWrite(locks: Seq[AnyRef]) extends ThreadState
previous-exams/2021-final-solutions/f3/src/test/scala/f3/instrumentation/Scheduler.scala deleted 100644 → 0
View file @ 071256c3
package f3.instrumentation
import java.util.concurrent._;
import scala.concurrent.duration._
import scala.collection.mutable._
import Stats._
import java.util.concurrent.atomic.AtomicInteger
sealed abstract class Result
case class RetVal(rets: List[Any]) extends Result
case class Except(msg: String, stackTrace: Array[StackTraceElement]) extends Result
case class Timeout(msg: String) extends Result
/**
* A class that maintains schedule and a set of thread ids.
* The schedules are advanced after an operation of a SchedulableBuffer is performed.
* Note: the real schedule that is executed may deviate from the input schedule
* due to the adjustments that had to be made for locks
*/
class Scheduler(sched: List[Int]) {
val maxOps = 500 // a limit on the maximum number of operations the code is allowed to perform
private var schedule = sched
private var numThreads = 0
private val realToFakeThreadId = Map[Long, Int]()
private val opLog = ListBuffer[String]() // a mutable list (used for efficient concat)
private val threadStates = Map[Int, ThreadState]()
/**
* Runs a set of operations in parallel as per the schedule.
* Each operation may consist of many primitive operations like reads or writes
* to shared data structure each of which should be executed using the function `exec`.
* @timeout in milliseconds
* @return true - all threads completed on time, false -some tests timed out.
*/
def runInParallel(timeout: Long, ops: List[() => Any]): Result = {
numThreads = ops.length
val threadRes = Array.fill(numThreads) { None: Any }
var exception: Option[Except] = None
val syncObject = new Object()
var completed = new AtomicInteger(0)
// create threads
val threads = ops.zipWithIndex.map {
case (op, i) =>
new Thread(new Runnable() {
def run(): Unit = {
val fakeId = i + 1
setThreadId(fakeId)
try {
updateThreadState(Start)
val res = op()
updateThreadState(End)
threadRes(i) = res
// notify the master thread if all threads have completed
if (completed.incrementAndGet() == ops.length) {
syncObject.synchronized { syncObject.notifyAll() }
}
} catch {
case e: Throwable if exception != None => // do nothing here and silently fail
case e: Throwable =>
log(s"throw ${e.toString}")
exception = Some(Except(s"Thread $fakeId crashed on the following schedule: \n" + opLog.mkString("\n"),
e.getStackTrace))
syncObject.synchronized { syncObject.notifyAll() }
//println(s"$fakeId: ${e.toString}")
//Runtime.getRuntime().halt(0) //exit the JVM and all running threads (no other way to kill other threads)
}
}
})
}
// start all threads
threads.foreach(_.start())
// wait for all threads to complete, or for an exception to be thrown, or for the time out to expire
var remTime = timeout
syncObject.synchronized {
timed { if(completed.get() != ops.length) syncObject.wait(timeout) } { time => remTime -= time }
}
if (exception.isDefined) {
exception.get
} else if (remTime <= 1) { // timeout ? using 1 instead of zero to allow for some errors
Timeout(opLog.mkString("\n"))
} else {
// every thing executed normally
RetVal(threadRes.toList)
}
}
// Updates the state of the current thread
def updateThreadState(state: ThreadState): Unit = {
val tid = threadId
synchronized {
threadStates(tid) = state
}
state match {
case Sync(lockToAquire, locks) =>
if (locks.indexOf(lockToAquire) < 0) waitForTurn else {
// Re-aqcuiring the same lock
updateThreadState(Running(lockToAquire +: locks))
}
case Start => waitStart()
case End => removeFromSchedule(tid)
case Running(_) =>
case _ => waitForTurn // Wait, SyncUnique, VariableReadWrite
}
}
def waitStart(): Unit = {
//while (threadStates.size < numThreads) {
//Thread.sleep(1)
//}
synchronized {
if (threadStates.size < numThreads) {
wait()
} else {
notifyAll()
}
}
}
def threadLocks = {
synchronized {
threadStates(threadId).locks
}
}
def threadState = {
synchronized {
threadStates(threadId)
}
}
def mapOtherStates(f: ThreadState => ThreadState) = {
val exception = threadId
synchronized {
for (k <- threadStates.keys if k != exception) {
threadStates(k) = f(threadStates(k))
}
}
}
def log(str: String) = {
if((realToFakeThreadId contains Thread.currentThread().getId())) {
val space = (" " * ((threadId - 1) * 2))
val s = space + threadId + ":" + "\n".r.replaceAllIn(str, "\n" + space + " ")
opLog += s
}
}
/**
* Executes a read or write operation to a global data structure as per the given schedule
* @param msg a message corresponding to the operation that will be logged
*/
def exec[T](primop: => T)(msg: => String, postMsg: => Option[T => String] = None): T = {
if(! (realToFakeThreadId contains Thread.currentThread().getId())) {
primop
} else {
updateThreadState(VariableReadWrite(threadLocks))
val m = msg
if(m != "") log(m)
if (opLog.size > maxOps)
throw new Exception(s"Total number of reads/writes performed by threads exceed $maxOps. A possible deadlock!")
val res = primop
postMsg match {
case Some(m) => log(m(res))
case None =>
}
res
}
}
private def setThreadId(fakeId: Int) = synchronized {
realToFakeThreadId(Thread.currentThread.getId) = fakeId
}
def threadId =
try {
realToFakeThreadId(Thread.currentThread().getId())
} catch {
case e: NoSuchElementException =>
throw new Exception("You are accessing shared variables in the constructor. This is not allowed. The variables are already initialized!")
}
private def isTurn(tid: Int) = synchronized {
(!schedule.isEmpty && schedule.head != tid)
}
def canProceed(): Boolean = {
val tid = threadId
canContinue match {
case Some((i, state)) if i == tid =>
//println(s"$tid: Runs ! Was in state $state")
canContinue = None
state match {
case Sync(lockToAquire, locks) => updateThreadState(Running(lockToAquire +: locks))
case SyncUnique(lockToAquire, locks) =>
mapOtherStates {
_ match {
case SyncUnique(lockToAquire2, locks2) if lockToAquire2 == lockToAquire => Wait(lockToAquire2, locks2)
case e => e
}
}
updateThreadState(Running(lockToAquire +: locks))
case VariableReadWrite(locks) => updateThreadState(Running(locks))
}
true
case Some((i, state)) =>
//println(s"$tid: not my turn but $i !")
false
case None =>
false
}
}
var threadPreference = 0 // In the case the schedule is over, which thread should have the preference to execute.
/** returns true if the thread can continue to execute, and false otherwise */
def decide(): Option[(Int, ThreadState)] = {
if (!threadStates.isEmpty) { // The last thread who enters the decision loop takes the decision.
//println(s"$threadId: I'm taking a decision")
if (threadStates.values.forall { case e: Wait => true case _ => false }) {
val waiting = threadStates.keys.map(_.toString).mkString(", ")
val s = if (threadStates.size > 1) "s" else ""
val are = if (threadStates.size > 1) "are" else "is"
throw new Exception(s"Deadlock: Thread$s $waiting $are waiting but all others have ended and cannot notify them.")
} else {
// Threads can be in Wait, Sync, SyncUnique, and VariableReadWrite mode.
// Let's determine which ones can continue.
val notFree = threadStates.collect { case (id, state) => state.locks }.flatten.toSet
val threadsNotBlocked = threadStates.toSeq.filter {
case (id, v: VariableReadWrite) => true
case (id, v: CanContinueIfAcquiresLock) => !notFree(v.lockToAquire) || (v.locks contains v.lockToAquire)
case _ => false
}
if (threadsNotBlocked.isEmpty) {
val waiting = threadStates.keys.map(_.toString).mkString(", ")
val s = if (threadStates.size > 1) "s" else ""
val are = if (threadStates.size > 1) "are" else "is"
val whoHasLock = threadStates.toSeq.flatMap { case (id, state) => state.locks.map(lock => (lock, id)) }.toMap
val reason = threadStates.collect {
case (id, state: CanContinueIfAcquiresLock) if !notFree(state.lockToAquire) =>
s"Thread $id is waiting on lock ${state.lockToAquire} held by thread ${whoHasLock(state.lockToAquire)}"
}.mkString("\n")
throw new Exception(s"Deadlock: Thread$s $waiting are interlocked. Indeed:\n$reason")
} else if (threadsNotBlocked.size == 1) { // Do not consume the schedule if only one thread can execute.
Some(threadsNotBlocked(0))
} else {
val next = schedule.indexWhere(t => threadsNotBlocked.exists { case (id, state) => id == t })
if (next != -1) {
//println(s"$threadId: schedule is $schedule, next chosen is ${schedule(next)}")
val chosenOne = schedule(next) // TODO: Make schedule a mutable list.
schedule = schedule.take(next) ++ schedule.drop(next + 1)
Some((chosenOne, threadStates(chosenOne)))
} else {
threadPreference = (threadPreference + 1) % threadsNotBlocked.size
val chosenOne = threadsNotBlocked(threadPreference) // Maybe another strategy
Some(chosenOne)
//threadsNotBlocked.indexOf(threadId) >= 0
/*
val tnb = threadsNotBlocked.map(_._1).mkString(",")
val s = if (schedule.isEmpty) "empty" else schedule.mkString(",")
val only = if (schedule.isEmpty) "" else " only"
throw new Exception(s"The schedule is $s but$only threads ${tnb} can continue")*/
}
}
}
} else canContinue
}
/**
* This will be called before a schedulable operation begins.
* This should not use synchronized
*/
var numThreadsWaiting = new AtomicInteger(0)
//var waitingForDecision = Map[Int, Option[Int]]() // Mapping from thread ids to a number indicating who is going to make the choice.
var canContinue: Option[(Int, ThreadState)] = None // The result of the decision thread Id of the thread authorized to continue.
private def waitForTurn = {
synchronized {
if (numThreadsWaiting.incrementAndGet() == threadStates.size) {
canContinue = decide()
notifyAll()
}
//waitingForDecision(threadId) = Some(numThreadsWaiting)
//println(s"$threadId Entering waiting with ticket number $numThreadsWaiting/${waitingForDecision.size}")
while (!canProceed()) wait()
}
numThreadsWaiting.decrementAndGet()
}
/**
* To be invoked when a thread is about to complete
*/
private def removeFromSchedule(fakeid: Int) = synchronized {
//println(s"$fakeid: I'm taking a decision because I finished")
schedule = schedule.filterNot(_ == fakeid)
threadStates -= fakeid
if (numThreadsWaiting.get() == threadStates.size) {
canContinue = decide()
notifyAll()
}
}
def getOperationLog() = opLog
}
previous-exams/2021-final-solutions/f3/src/test/scala/f3/instrumentation/Stats.scala deleted 100644 → 0
View file @ 071256c3
/* Copyright 2009-2015 EPFL, Lausanne */
package f3.instrumentation
import java.lang.management._
/**
* A collection of methods that can be used to collect run-time statistics about Leon programs.
* This is mostly used to test the resources properties of Leon programs
*/
object Stats {
def timed[T](code: => T)(cont: Long => Unit): T = {
var t1 = System.currentTimeMillis()
val r = code
cont((System.currentTimeMillis() - t1))
r
}
def withTime[T](code: => T): (T, Long) = {
var t1 = System.currentTimeMillis()
val r = code
(r, (System.currentTimeMillis() - t1))
}
}
previous-exams/2021-final-solutions/f3/src/test/scala/f3/instrumentation/TestHelper.scala deleted 100644 → 0
View file @ 071256c3
package f3.instrumentation
import scala.util.Random
import scala.collection.mutable.{Map => MutableMap}
import Stats._
object TestHelper {
val noOfSchedules = 10000 // set this to 100k during deployment
val readWritesPerThread = 20 // maximum number of read/writes possible in one thread
val contextSwitchBound = 10
val testTimeout = 150 // the total time out for a test in seconds
val schedTimeout = 15 // the total time out for execution of a schedule in secs
// Helpers
/*def testManySchedules(op1: => Any): Unit = testManySchedules(List(() => op1))
def testManySchedules(op1: => Any, op2: => Any): Unit = testManySchedules(List(() => op1, () => op2))
def testManySchedules(op1: => Any, op2: => Any, op3: => Any): Unit = testManySchedules(List(() => op1, () => op2, () => op3))
def testManySchedules(op1: => Any, op2: => Any, op3: => Any, op4: => Any): Unit = testManySchedules(List(() => op1, () => op2, () => op3, () => op4))*/
def testSequential[T](ops: Scheduler => Any)(assertions: T => (Boolean, String)) =
testManySchedules(1,
(sched: Scheduler) => {
(List(() => ops(sched)),
(res: List[Any]) => assertions(res.head.asInstanceOf[T]))
})
/**
* @numThreads number of threads
* @ops operations to be executed, one per thread
* @assertion as condition that will executed after all threads have completed (without exceptions)
* the arguments are the results of the threads
*/
def testManySchedules(numThreads: Int,
ops: Scheduler =>
(List[() => Any], // Threads
List[Any] => (Boolean, String)) // Assertion
) = {
var timeout = testTimeout * 1000L
val threadIds = (1 to numThreads)
//(1 to scheduleLength).flatMap(_ => threadIds).toList.permutations.take(noOfSchedules).foreach {
val schedules = (new ScheduleGenerator(numThreads)).schedules()
var schedsExplored = 0
schedules.takeWhile(_ => schedsExplored <= noOfSchedules && timeout > 0).foreach {
//case _ if timeout <= 0 => // break
case schedule =>
schedsExplored += 1
val schedr = new Scheduler(schedule)
//println("Exploring Sched: "+schedule)
val (threadOps, assertion) = ops(schedr)
if (threadOps.size != numThreads)
throw new IllegalStateException(s"Number of threads: $numThreads, do not match operations of threads: $threadOps")
timed { schedr.runInParallel(schedTimeout * 1000, threadOps) } { t => timeout -= t } match {
case Timeout(msg) =>
throw new java.lang.AssertionError("assertion failed\n"+"The schedule took too long to complete. A possible deadlock! \n"+msg)
case Except(msg, stkTrace) =>
val traceStr = "Thread Stack trace: \n"+stkTrace.map(" at "+_.toString).mkString("\n")
throw new java.lang.AssertionError("assertion failed\n"+msg+"\n"+traceStr)
case RetVal(threadRes) =>
// check the assertion
val (success, custom_msg) = assertion(threadRes)
if (!success) {
val msg = "The following schedule resulted in wrong results: \n" + custom_msg + "\n" + schedr.getOperationLog().mkString("\n")
throw new java.lang.AssertionError("Assertion failed: "+msg)
}
}
}
if (timeout <= 0) {
throw new java.lang.AssertionError("Test took too long to complete! Cannot check all schedules as your code is too slow!")
}
}
/**
* A schedule generator that is based on the context bound
*/
class ScheduleGenerator(numThreads: Int) {
val scheduleLength = readWritesPerThread * numThreads
val rands = (1 to scheduleLength).map(i => new Random(0xcafe * i)) // random numbers for choosing a thread at each position
def schedules(): LazyList[List[Int]] = {
var contextSwitches = 0
var contexts = List[Int]() // a stack of thread ids in the order of context-switches
val remainingOps = MutableMap[Int, Int]()
remainingOps ++= (1 to numThreads).map(i => (i, readWritesPerThread)) // num ops remaining in each thread
val liveThreads = (1 to numThreads).toSeq.toBuffer
/**
* Updates remainingOps and liveThreads once a thread is chosen for a position in the schedule
*/
def updateState(tid: Int): Unit = {
val remOps = remainingOps(tid)
if (remOps == 0) {
liveThreads -= tid
} else {
remainingOps += (tid -> (remOps - 1))
}
}
val schedule = rands.foldLeft(List[Int]()) {
case (acc, r) if contextSwitches < contextSwitchBound =>
val tid = liveThreads(r.nextInt(liveThreads.size))
contexts match {
case prev :: tail if prev != tid => // we have a new context switch here
contexts +:= tid
contextSwitches += 1
case prev :: tail =>
case _ => // init case
contexts +:= tid
}
updateState(tid)
acc :+ tid
case (acc, _) => // here context-bound has been reached so complete the schedule without any more context switches
if (!contexts.isEmpty) {
contexts = contexts.dropWhile(remainingOps(_) == 0)
}
val tid = contexts match {
case top :: tail => top
case _ => liveThreads(0) // here, there has to be threads that have not even started
}
updateState(tid)
acc :+ tid
}
schedule #:: schedules()
}
}
}
previous-exams/2021-final-solutions/f3/src/test/scala/f3/instrumentation/TestUtils.scala deleted 100644 → 0
View file @ 071256c3
package f3.instrumentation
import scala.concurrent._
import scala.concurrent.duration._
import scala.concurrent.ExecutionContext.Implicits.global
object TestUtils {
def failsOrTimesOut[T](action: => T): Boolean = {
val asyncAction = Future {
action
}
try {
Await.result(asyncAction, 2000.millisecond)
} catch {
case _: Throwable => return true
}
return false
}
}
previous-exams/2021-final-solutions/f3/src/test/scala/f3/overrides.scala deleted 100644 → 0
View file @ 071256c3
package f3
import instrumentation._
class SchedulableThreadMap[A](val scheduler: Scheduler) extends ThreadMap[A] with MockedMonitor:
override def currentThreadHasValue: Boolean = scheduler.exec {
super.currentThreadHasValue
} ("", Some(res => s"currentThreadHasValue is $res"))
override def currentThreadValue: Option[A] = scheduler.exec {
super.currentThreadValue
} ("", Some(res => s"currentThreadValue is $res"))
override def setCurrentThreadValue(value: A): Unit = scheduler.exec {
super.setCurrentThreadValue(value)
} (s"setCurrentThreadValue($value)")
override def deleteCurrentThreadValue(): Unit = scheduler.exec {
super.deleteCurrentThreadValue()
} ("deleteCurrentThreadValue()")
override def waitForall(predicate: A => Boolean): Unit = scheduler.exec {
super.waitForall(predicate)
} ("waitForall")
def allValues: List[A] = synchronized {
theMap.values.toList
}
end SchedulableThreadMap
class SchedulableRCU(scheduler: Scheduler) extends RCU with LockFreeMonitor:
override protected val latestVersion = SchedulableAtomicLong(0, scheduler, "latestVersion")
override protected val readersVersion: ThreadMap[Long] = SchedulableThreadMap(scheduler)
class SchedulableInMemoryFileSystem(scheduler: Scheduler) extends InMemoryFileSystem:
override def createFile(file: FileName, content: String): Unit = scheduler.exec {
super.createFile(file, content)
} (s"createFile($file)")
override def readFile(file: FileName): String = scheduler.exec {
super.readFile(file)
} (s"readFile($file)")
override def deleteFile(file: FileName): Unit = scheduler.exec {
super.deleteFile(file)
} (s"deleteFile($file)")
class SchedulableUpdateServer(scheduler: Scheduler, fs: InMemoryFileSystem) extends UpdateServer(fs) with LockFreeMonitor:
override val rcu = SchedulableRCU(scheduler)
class SchedulableAtomicLong(initial: Long, scheduler: Scheduler, name: String) extends AtomicLong(initial):
override def get: Long = scheduler.exec {
super.get
} (s"", Some(res => s"$name: get $res"))
override def set(value: Long): Unit = scheduler.exec {
super.set(value)
} (s"$name: set $value", None)
override def incrementAndGet(): Long = scheduler.exec {
super.incrementAndGet()
} (s"", Some(res => s"$name: incrementAndGet $res"))
override def getAndIncrement(): Long = scheduler.exec {
super.getAndIncrement()
} (s"", Some(res => s"$name: getandIncrement $res"))
override def compareAndSet(expected: Long, newValue: Long): Boolean =
scheduler.exec {
super.compareAndSet(expected, newValue)
} (s"$name: compareAndSet(expected = $expected, newValue = $newValue)",
Some(res => s"$name: Did it set? $res") )
end SchedulableAtomicLong
class SchedulableAtomicReference[T](initial: T, scheduler: Scheduler, name: String) extends AtomicReference(initial):
override def get: T = scheduler.exec {
super.get
} (s"", Some(res => s"$name: get $res"))
override def set(value: T): Unit = scheduler.exec {
super.set(value)
} (s"$name: set $value", None)
previous-exams/2021-final-solutions/f4/.gitignore deleted 100644 → 0
View file @ 071256c3
# General
*.DS_Store
*.swp
*~
# Dotty
*.class
*.tasty
*.hasTasty
# sbt
target/
# IDE
.bsp
.bloop
.metals
.vscode
# datasets
stackoverflow-grading.csv
wikipedia-grading.dat
previous-exams/2021-final-solutions/f4/assignment.sbt deleted 100644 → 0
View file @ 071256c3
// Student tasks (i.e. submit, packageSubmission)
enablePlugins(StudentTasks)
previous-exams/2021-final-solutions/f4/build.sbt deleted 100644 → 0
View file @ 071256c3
course := "final"
assignment := "f4"
scalaVersion := "3.0.0-RC1"
scalacOptions ++= Seq("-language:implicitConversions", "-deprecation")
libraryDependencies ++= Seq(
("org.apache.spark" %% "spark-core" % "3.2.0-SNAPSHOT").withDottyCompat(scalaVersion.value),
)
// Contains Spark 3 snapshot built against 2.13: https://github.com/smarter/spark/tree/scala-2.13
resolvers += "Spark Snapshots Copy" at "https://scala-webapps.epfl.ch/artifactory/spark-snapshot/"
libraryDependencies += "org.scalameta" %% "munit" % "0.7.22"
val MUnitFramework = new TestFramework("munit.Framework")
testFrameworks += MUnitFramework
// Decode Scala names
testOptions += Tests.Argument(MUnitFramework, "-s")
testSuite := "f4.F4Suite"
// Without forking, ctrl-c doesn't actually fully stop Spark
fork in run := true
fork in Test := true
previous-exams/2021-final-solutions/f4/grading-tests.jar deleted 100644 → 0
View file @ 071256c3
File deleted
previous-exams/2021-final-solutions/f4/project/FilteringReporterPlugin.scala deleted 100644 → 0
View file @ 071256c3
package sbt // To access the private[sbt] compilerReporter key
package filteringReporterPlugin
import Keys._
import ch.epfl.lamp._
object FilteringReporterPlugin extends AutoPlugin {
override lazy val projectSettings = Seq(
// Turn off warning coming from scalameter that we cannot fix without changing scalameter
compilerReporter in (Compile, compile) ~= { reporter => new FilteringReporter(reporter) }
)
}
class FilteringReporter(reporter: xsbti.Reporter) extends xsbti.Reporter {
def reset(): Unit = reporter.reset()
def hasErrors: Boolean = reporter.hasErrors
def hasWarnings: Boolean = reporter.hasWarnings
def printSummary(): Unit = reporter.printSummary()
def problems: Array[xsbti.Problem] = reporter.problems
def log(problem: xsbti.Problem): Unit = {
if (!problem.message.contains("An existential type that came from a Scala-2 classfile cannot be"))
reporter.log(problem)
}
def comment(pos: xsbti.Position, msg: String): Unit =
reporter.comment(pos, msg)
override def toString = s"CollectingReporter($reporter)"
}
previous-exams/2021-final-solutions/f4/project/MOOCSettings.scala deleted 100644 → 0
View file @ 071256c3
package ch.epfl.lamp
import sbt._
import sbt.Keys._
/**
* Coursera uses two versions of each assignment. They both have the same assignment key and part id but have
* different item ids.
*
* @param key Assignment key
* @param partId Assignment partId
* @param itemId Item id of the non premium version
* @param premiumItemId Item id of the premium version (`None` if the assignment is optional)
*/
case class CourseraId(key: String, partId: String, itemId: String, premiumItemId: Option[String])
/**
* Settings shared by all assignments, reused in various tasks.
*/
object MOOCSettings extends AutoPlugin {
override def requires = super.requires && filteringReporterPlugin.FilteringReporterPlugin
object autoImport {
val course = SettingKey[String]("course")
val assignment = SettingKey[String]("assignment")
val options = SettingKey[Map[String, Map[String, String]]]("options")
val courseraId = settingKey[CourseraId]("Coursera-specific information identifying the assignment")
val testSuite = settingKey[String]("Fully qualified name of the test suite of this assignment")
.withRank(KeyRanks.Invisible)
// Convenient alias
type CourseraId = ch.epfl.lamp.CourseraId
val CourseraId = ch.epfl.lamp.CourseraId
}
import autoImport._
override val globalSettings: Seq[Def.Setting[_]] = Seq(
// supershell is verbose, buggy and useless.
useSuperShell := false
)
override val projectSettings: Seq[Def.Setting[_]] = Seq(
parallelExecution in Test := false,
// Report test result after each test instead of waiting for every test to finish
logBuffered in Test := false,
name := s"${course.value}-${assignment.value}"
)
}
previous-exams/2021-final-solutions/f4/project/StudentTasks.scala deleted 100644 → 0
View file @ 071256c3
package ch.epfl.lamp
import sbt._
import Keys._
// import scalaj.http._
import java.io.{File, FileInputStream, IOException}
import org.apache.commons.codec.binary.Base64
// import play.api.libs.json.{Json, JsObject, JsPath}
import scala.util.{Failure, Success, Try}
/**
* Provides tasks for submitting the assignment
*/
object StudentTasks extends AutoPlugin {
override def requires = super.requires && MOOCSettings
object autoImport {
val packageSourcesOnly = TaskKey[File]("packageSourcesOnly", "Package the sources of the project")
val packageBinWithoutResources = TaskKey[File]("packageBinWithoutResources", "Like packageBin, but without the resources")
val packageSubmissionZip = TaskKey[File]("packageSubmissionZip")
val packageSubmission = inputKey[Unit]("package solution as an archive file")
lazy val Grading = config("grading") extend(Runtime)
}
import autoImport._
import MOOCSettings.autoImport._
override lazy val projectSettings = Seq(
packageSubmissionSetting,
fork := true,
connectInput in run := true,
outputStrategy := Some(StdoutOutput),
) ++
packageSubmissionZipSettings ++
inConfig(Grading)(Defaults.testSettings ++ Seq(
unmanagedJars += file("grading-tests.jar"),
definedTests := (definedTests in Test).value,
internalDependencyClasspath := (internalDependencyClasspath in Test).value
))
/** **********************************************************
* SUBMITTING A SOLUTION TO COURSERA
*/
val packageSubmissionZipSettings = Seq(
packageSubmissionZip := {
val submission = crossTarget.value / "submission.zip"
val sources = (packageSourcesOnly in Compile).value
val binaries = (packageBinWithoutResources in Compile).value
IO.zip(Seq(sources -> "sources.zip", binaries -> "binaries.jar"), submission, None)
submission
},
artifactClassifier in packageSourcesOnly := Some("sources"),
artifact in (Compile, packageBinWithoutResources) ~= (art => art.withName(art.name + "-without-resources"))
) ++
inConfig(Compile)(
Defaults.packageTaskSettings(packageSourcesOnly, Defaults.sourceMappings) ++
Defaults.packageTaskSettings(packageBinWithoutResources, Def.task {
val relativePaths =
(unmanagedResources in Compile).value.flatMap(Path.relativeTo((unmanagedResourceDirectories in Compile).value)(_))
(mappings in (Compile, packageBin)).value.filterNot { case (_, path) => relativePaths.contains(path) }
})
)
val maxSubmitFileSize = {
val mb = 1024 * 1024
10 * mb
}
/** Check that the jar exists, isn't empty, isn't crazy big, and can be read
* If so, encode jar as base64 so we can send it to Coursera
*/
def prepareJar(jar: File, s: TaskStreams): String = {
val errPrefix = "Error submitting assignment jar: "
val fileLength = jar.length()
if (!jar.exists()) {
s.log.error(errPrefix + "jar archive does not exist\n" + jar.getAbsolutePath)
failSubmit()
} else if (fileLength == 0L) {
s.log.error(errPrefix + "jar archive is empty\n" + jar.getAbsolutePath)
failSubmit()
} else if (fileLength > maxSubmitFileSize) {
s.log.error(errPrefix + "jar archive is too big. Allowed size: " +
maxSubmitFileSize + " bytes, found " + fileLength + " bytes.\n" +
jar.getAbsolutePath)
failSubmit()
} else {
val bytes = new Array[Byte](fileLength.toInt)
val sizeRead = try {
val is = new FileInputStream(jar)
val read = is.read(bytes)
is.close()
read
} catch {
case ex: IOException =>
s.log.error(errPrefix + "failed to read sources jar archive\n" + ex.toString)
failSubmit()
}
if (sizeRead != bytes.length) {
s.log.error(errPrefix + "failed to read the sources jar archive, size read: " + sizeRead)
failSubmit()
} else encodeBase64(bytes)
}
}
/** Task to package solution to a given file path */
lazy val packageSubmissionSetting = packageSubmission := {
val args: Seq[String] = Def.spaceDelimited("[path]").parsed
val s: TaskStreams = streams.value // for logging
val jar = (packageSubmissionZip in Compile).value
val base64Jar = prepareJar(jar, s)
val path = args.headOption.getOrElse((baseDirectory.value / "submission.jar").absolutePath)
scala.tools.nsc.io.File(path).writeAll(base64Jar)
}
/*
/** Task to submit a solution to coursera */
val submit = inputKey[Unit]("submit solution to Coursera")
lazy val submitSetting = submit := {
// Fail if scalafix linting does not pass.
scalafixLinting.value
val args: Seq[String] = Def.spaceDelimited("<arg>").parsed
val s: TaskStreams = streams.value // for logging
val jar = (packageSubmissionZip in Compile).value
val assignmentDetails =
courseraId.?.value.getOrElse(throw new MessageOnlyException("This assignment can not be submitted to Coursera because the `courseraId` setting is undefined"))
val assignmentKey = assignmentDetails.key
val courseName =
course.value match {
case "capstone" => "scala-capstone"
case "bigdata" => "scala-spark-big-data"
case other => other
}
val partId = assignmentDetails.partId
val itemId = assignmentDetails.itemId
val premiumItemId = assignmentDetails.premiumItemId
val (email, secret) = args match {
case email :: secret :: Nil =>
(email, secret)
case _ =>
val inputErr =
s"""|Invalid input to `submit`. The required syntax for `submit` is:
|submit <email-address> <submit-token>
|
|The submit token is NOT YOUR LOGIN PASSWORD.
|It can be obtained from the assignment page:
|https://www.coursera.org/learn/$courseName/programming/$itemId
|${
premiumItemId.fold("") { id =>
s"""or (for premium learners):
|https://www.coursera.org/learn/$courseName/programming/$id
""".stripMargin
}
}
""".stripMargin
s.log.error(inputErr)
failSubmit()
}
val base64Jar = prepareJar(jar, s)
val json =
s"""|{
| "assignmentKey":"$assignmentKey",
| "submitterEmail":"$email",
| "secret":"$secret",
| "parts":{
| "$partId":{
| "output":"$base64Jar"
| }
| }
|}""".stripMargin
def postSubmission[T](data: String): Try[HttpResponse[String]] = {
val http = Http("https://www.coursera.org/api/onDemandProgrammingScriptSubmissions.v1")
val hs = List(
("Cache-Control", "no-cache"),
("Content-Type", "application/json")
)
s.log.info("Connecting to Coursera...")
val response = Try(http.postData(data)
.headers(hs)
.option(HttpOptions.connTimeout(10000)) // scalaj default timeout is only 100ms, changing that to 10s
.asString) // kick off HTTP POST
response
}
val connectMsg =
s"""|Attempting to submit "${assignment.value}" assignment in "$courseName" course
|Using:
|- email: $email
|- submit token: $secret""".stripMargin
s.log.info(connectMsg)
def reportCourseraResponse(response: HttpResponse[String]): Unit = {
val code = response.code
val respBody = response.body
/* Sample JSON response from Coursera
{
"message": "Invalid email or token.",
"details": {
"learnerMessage": "Invalid email or token."
}
}
*/
// Success, Coursera responds with 2xx HTTP status code
if (response.is2xx) {
val successfulSubmitMsg =
s"""|Successfully connected to Coursera. (Status $code)
|
|Assignment submitted successfully!
|
|You can see how you scored by going to:
|https://www.coursera.org/learn/$courseName/programming/$itemId/
|${
premiumItemId.fold("") { id =>
s"""or (for premium learners):
|https://www.coursera.org/learn/$courseName/programming/$id
""".stripMargin
}
}
|and clicking on "My Submission".""".stripMargin
s.log.info(successfulSubmitMsg)
}
// Failure, Coursera responds with 4xx HTTP status code (client-side failure)
else if (response.is4xx) {
val result = Try(Json.parse(respBody)).toOption
val learnerMsg = result match {
case Some(resp: JsObject) =>
(JsPath \ "details" \ "learnerMessage").read[String].reads(resp).get
case Some(x) => // shouldn't happen
"Could not parse Coursera's response:\n" + x
case None =>
"Could not parse Coursera's response:\n" + respBody
}
val failedSubmitMsg =
s"""|Submission failed.
|There was something wrong while attempting to submit.
|Coursera says:
|$learnerMsg (Status $code)""".stripMargin
s.log.error(failedSubmitMsg)
}
// Failure, Coursera responds with 5xx HTTP status code (server-side failure)
else if (response.is5xx) {
val failedSubmitMsg =
s"""|Submission failed.
|Coursera seems to be unavailable at the moment (Status $code)
|Check https://status.coursera.org/ and try again in a few minutes.
""".stripMargin
s.log.error(failedSubmitMsg)
}
// Failure, Coursera repsonds with an unexpected status code
else {
val failedSubmitMsg =
s"""|Submission failed.
|Coursera replied with an unexpected code (Status $code)
""".stripMargin
s.log.error(failedSubmitMsg)
}
}
// kick it all off, actually make request
postSubmission(json) match {
case Success(resp) => reportCourseraResponse(resp)
case Failure(e) =>
val failedConnectMsg =
s"""|Connection to Coursera failed.
|There was something wrong while attempting to connect to Coursera.
|Check your internet connection.
|${e.toString}""".stripMargin
s.log.error(failedConnectMsg)
}
}
*/
def failSubmit(): Nothing = {
sys.error("Submission failed")
}
/**
* *****************
* DEALING WITH JARS
*/
def encodeBase64(bytes: Array[Byte]): String =
new String(Base64.encodeBase64(bytes))
}
previous-exams/2021-final-solutions/f4/project/build.properties deleted 100644 → 0
View file @ 071256c3
sbt.version=1.4.7
previous-exams/2021-final-solutions/f4/project/buildSettings.sbt deleted 100644 → 0
View file @ 071256c3
// Used for Coursera submission (StudentPlugin)
// libraryDependencies += "org.scalaj" %% "scalaj-http" % "2.4.2"
// libraryDependencies += "com.typesafe.play" %% "play-json" % "2.7.4"
// Used for Base64 (StudentPlugin)
libraryDependencies += "commons-codec" % "commons-codec" % "1.10"
previous-exams/2021-final-solutions/f4/project/plugins.sbt deleted 100644 → 0
View file @ 071256c3
// addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.28")
addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.8")
addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % "0.5.3")
previous-exams/2021-final-solutions/f4/src/main/resources/f4/shakespeare.txt deleted 100644 → 0
View file @ 071256c3
Source diff could not be displayed: it is too large. Options to address this: view the blob.
previous-exams/2021-final-solutions/f4/src/main/scala/f4/F4Data.scala deleted 100644 → 0
View file @ 071256c3
package f4
import scala.io.Source
import scala.io.Codec
object F4Data {
val punctuationRx = "[,:;?!.'\"]+".r
def linesIterator: Iterator[String] = {
Option(getClass.getResourceAsStream("/f4/shakespeare.txt")) match {
case Some(resource) =>
DoubleLineIterator(Source.fromInputStream(resource)(Codec.UTF8))
case None =>
throw new RuntimeException("The resource with the corpus is unexpectedly missing, please inform the staff.")
}
}
def lines = linesIterator.toSeq
}
class DoubleLineIterator(underlying: Iterator[Char]) extends scala.collection.AbstractIterator[String] with Iterator[String] {
private[this] val sb = new StringBuilder
lazy val iter: BufferedIterator[Char] = underlying.buffered
def getc(): Boolean = iter.hasNext && {
val ch = iter.next()
if (ch == '\n') {
val has = iter.hasNext
val ch2 = if has then iter.next() else ' '
if (has && ch2 == '\n')
false
else {
sb append ' '
sb append ch2
true
}
} else {
sb append ch
true
}
}
def hasNext: Boolean = iter.hasNext
def next(): String = {
sb.clear()
while (getc()) { }
sb.toString
}
}
previous-exams/2021-final-solutions/f4/src/main/scala/f4/NGrams.scala deleted 100644 → 0
View file @ 071256c3
package f4
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.log4j.{Logger, Level}
import org.apache.spark.rdd.RDD
import scala.util.Properties.isWin
/** This class calculates word n-gram frequencies in the text it receives, but
* the implementation is faulty and the result is calculated locally.
*
* Your tasks will revolve around fixing the implementation, making the
* calculation distributed and finally, calculating per-speaker word n-gram
* frequencies.
*/
abstract class NGrams {
type Triple[T] = (T, T, T)
type NGramMap = Map[(String, String, String), Int]
val sc: SparkContext
val lines: Seq[String]
val punctuationRx = "[-,:;?!.'\"]+".r
def removePunctuation(str: String) = punctuationRx.replaceAllIn(str, "")
def splitIntoWords(str: String) = str.trim.split("\\s+")
/** Splits a string into a sequence of n-grams.
*
* There are two issues with the implementation:
* [TASK 1]
* Capitalization of words should not matter.
* Convert all words in n-grams to lower case.
*
* [TASK 2]
* If a sentence has less than 3 words, "fake" words should be added to its end.
* For such sentences, add empty strings at the end.
*/
def splitIntoNGrams(line: String): List[Triple[String]] = {
val words = splitIntoWords(removePunctuation(line).toLowerCase)
words.sliding(3).map { seq =>
def at(i: Int) = if i < seq.length then seq(i) else ""
(at(0), at(1), at(2))
}.toList
}
def createNGramsRDD(lines: Seq[String]): RDD[List[Triple[String]]] =
sc.parallelize(lines).map(splitIntoNGrams)
/** This function _locally_ calculates the frequency of n-grams it receives. */
def localNGrams(ngrams: RDD[List[Triple[String]]]): NGramMap = {
ngrams.toLocalIterator
.flatMap(ngramSeq => ngramSeq.map { ng =>
Map(ng -> 1)
})
.reduce { (left, right) =>
right.foldLeft(left) {
case (left, (ng, weightA)) =>
left.updatedWith(ng)(weightBOpt => Some(weightA + weightBOpt.getOrElse(0)))
}
}
}
/** [TASK 3] Based on the above code, calculate n-gram frequency in a
* distributed manner. Define [[aggregateNGrams_zero]],
* [[aggregateNGrams_seqOp]], [[aggregateNGrams_combOp]] in such a way that
* this function will return the same result as the one above.
*/
final def aggregateNGrams(ngrams: RDD[List[Triple[String]]]): NGramMap =
ngrams.aggregate(aggregateNGrams_zero)(aggregateNGrams_seqOp, aggregateNGrams_combOp)
def aggregateNGrams_zero: NGramMap =
Map.empty[Triple[String], Int]
def aggregateNGrams_seqOp(acc: NGramMap, ngramSeq: List[Triple[String]]) =
ngramSeq.foldLeft(acc) {
case (acc, ng) =>
acc.updatedWith(ng)(w => Some(w.getOrElse(0) + 1))
}
def aggregateNGrams_combOp(acc: NGramMap, ngramMap: NGramMap) =
ngramMap.foldLeft(acc) {
case (acc, (ng, w)) =>
acc.updatedWith(ng)(ww => Some(w + ww.getOrElse(0)))
}
/** Your two final tasks are about using Spark to calculate the n-gram frequency
* per each speaker in the play.
*
* [TASK 4]
* To extract each line's speaker from the text, implement [[createSpeakerLinesRDD]].
*
* [TASK 5]
* To calculate the n-gram frequency, define [[createSpeakerLineNGramsRDD]]
* and [[createSpeakerNGramsRDD]].
*/
final def calculateSpeakerLineNGrams(lines: Seq[String]): Map[String, NGramMap] =
createSpeakerNGramsRDD(createSpeakerLineNGramsRDD(createSpeakerLinesRDD(lines)))
.toLocalIterator.toMap
/** Use [[lines]] to create an RDD of (Speaker, Line) pairs.
*
* Each line in [[lines]] looks like "SPEAKER: LINE". To separate the two,
* consider using the `.split` method on [[String]]. You can assume that the
* speaker's name doesn't contain a colon.
*/
def createSpeakerLinesRDD(lines: Seq[String]): RDD[(String, String)] =
sc.parallelize(lines)
.map { s =>
val Array(speaker, line) = s.split(":", 2)
assert(line != "", speaker)
speaker -> line
}
/** Takes the result of [[createSpeakerLinesRDD]] and calculates n-grams for each
* sentence, WITHOUT aggregating by speaker. Example:
* ```
* val input = Seq(
* "First Citizen" -> "Before we proceed! Before we proceed, hear me speak."
* "First Citizen" -> "Hear me speak."
* "SEBASTIAN" -> "Art thou waking?"
* )
*
* val output = Seq(
* "First Citizen" -> Map(
* ("before", "we", "proceed") -> 2,
* ("hear", "me", "speak") -> 1,
* // other n-grams following from the first sentence...
* ),
* "First Citizen" -> Map(
* ("hear", "me", "speak") -> 1,
* ),
* "SEBASTIAN" -> Map(
* ("art", "thou", "waking") -> 1,
* ),
* )
* ```
*/
def createSpeakerLineNGramsRDD(
speakerLines: RDD[(String, String)]
): RDD[(String, NGramMap)] =
speakerLines
.mapValues { s =>
splitIntoNGrams(s)
.map(List(_))
.foldLeft(aggregateNGrams_zero)(aggregateNGrams_seqOp)
}
/** Takes the result of [[createSpeakerLineNGramsRDD]] and aggregates the n-grams
* per each speaker. Example:
* ```
* val input = Seq(
* "First Citizen" -> Map(
* ("before", "we", "proceed") -> 1,
* ("hear", "me", "speak") -> 1,
* ),
* "First Citizen" -> Map(("hear", "me", "speak") -> 1),
* "SEBASTIAN" -> Map(("art", "thou", "waking") -> 1),
* "SEBASTIAN" -> Map(("art", "thou", "waking") -> 1),
* )
*
* val output = Seq(
* "First Citizen" -> Map(
* ("before", "we", "proceed") -> 1,
* ("hear", "me", "speak") -> 2,
* ),
* "SEBASTIAN" -> Map(("art", "thou", "waking") -> 2),
* )
* ```
*/
def createSpeakerNGramsRDD(
speakerLineNGrams: RDD[(String, NGramMap)]
): RDD[(String, NGramMap)] =
speakerLineNGrams
.groupByKey
.mapValues(_.reduce(aggregateNGrams_combOp))
}
object NGrams extends NGrams {
// Reduce Spark logging verbosity
Logger.getLogger("org").setLevel(Level.ERROR)
val conf: SparkConf = new SparkConf().setMaster("local[2]").setAppName("NGram")
val sc: SparkContext = new SparkContext(conf)
val lines = F4Data.lines
def main(args: Array[String]): Unit = {
timed("main", {
val rdd =
createSpeakerNGramsRDD(createSpeakerLineNGramsRDD(createSpeakerLinesRDD(lines)))
rdd.toLocalIterator.foreach { p =>
println(p)
}
})
}
val timing = new StringBuilder
def timed[T](label: String, code: => T): T = {
val start = System.currentTimeMillis()
val result = code
val stop = System.currentTimeMillis()
timing.append(s"Processing $label took ${stop - start} ms.\n")
result
}
}