Skip to content
Snippets Groups Projects
Commit e4ee3453 authored by Matt Bovel's avatar Matt Bovel
Browse files

Update final 2021 solutions

parent a2bbd8b7
Branches
No related tags found
No related merge requests found
Showing
with 631 additions and 295 deletions
# General
*.DS_Store
*.swp
*~
# Dotty
*.class
*.tasty
*.hasTasty
# sbt
target/
# IDE
.bsp
logs/
.bloop
.bsp
.dotty-ide-artifact
.dotty-ide.json
.idea
.metals
.vscode
# datasets
stackoverflow-grading.csv
wikipedia-grading.dat
*.csv
*.dat
metals.sbt
// Student tasks (i.e. submit, packageSubmission)
enablePlugins(StudentTasks)
assignmentVersion.withRank(KeyRanks.Invisible) := "eadbf7a6"
course := "final"
assignment := "f1"
scalaVersion := "3.0.0-RC1"
scalacOptions += "-nowarn"
course := "concpar"
assignment := "concpar21final01"
scalaVersion := "3.1.0"
scalacOptions ++= Seq("-language:implicitConversions")
libraryDependencies += "org.scalameta" %% "munit" % "1.0.0-M3" % Test
enablePlugins(PlayScala)
disablePlugins(PlayLayoutPlugin)
libraryDependencies := libraryDependencies.value.map(_.withDottyCompat(scalaVersion.value))
libraryDependencies += "org.scalameta" %% "munit" % "0.7.22"
libraryDependencies := libraryDependencies.value.map(dep =>
if(dep.organization == "com.typesafe.play") dep.cross(CrossVersion.for3Use2_13)
else dep
)
val MUnitFramework = new TestFramework("munit.Framework")
testFrameworks += MUnitFramework
// Decode Scala names
testOptions += Tests.Argument(MUnitFramework, "-s")
testSuite := "f1.F1Suite"
......@@ -2,143 +2,63 @@ package ch.epfl.lamp
import sbt._
import Keys._
// import scalaj.http._
import java.io.{File, FileInputStream, IOException}
import org.apache.commons.codec.binary.Base64
// import play.api.libs.json.{Json, JsObject, JsPath}
import scala.util.{Failure, Success, Try}
import scalaj.http._
import play.api.libs.json.{Json, JsObject, JsPath}
/**
* Provides tasks for submitting the assignment
*/
object StudentTasks extends AutoPlugin {
override def requires = super.requires && MOOCSettings
* Coursera uses two versions of each assignment. They both have the same assignment key and part id but have
* different item ids.
*
* @param key Assignment key
* @param partId Assignment partId
* @param itemId Item id of the non premium version
* @param premiumItemId Item id of the premium version (`None` if the assignment is optional)
*/
case class CourseraId(courseId: String, key: String, partId: String, itemId: String, premiumItemId: Option[String])
object autoImport {
val packageSourcesOnly = TaskKey[File]("packageSourcesOnly", "Package the sources of the project")
val packageBinWithoutResources = TaskKey[File]("packageBinWithoutResources", "Like packageBin, but without the resources")
val packageSubmissionZip = TaskKey[File]("packageSubmissionZip")
val packageSubmission = inputKey[Unit]("package solution as an archive file")
lazy val Grading = config("grading") extend(Runtime)
}
object CourseraStudent extends AutoPlugin {
override def requires = super.requires && MOOCSettings
import autoImport._
object autoImport {
val options = SettingKey[Map[String, Map[String, String]]]("options")
val courseraId = settingKey[CourseraId]("Coursera-specific information identifying the assignment")
// Convenient alias
type CourseraId = ch.epfl.lamp.CourseraId
val CourseraId = ch.epfl.lamp.CourseraId
}
import StudentTasks.autoImport._
import MOOCSettings.autoImport._
import autoImport._
override lazy val projectSettings = Seq(
packageSubmissionSetting,
fork := true,
connectInput in run := true,
outputStrategy := Some(StdoutOutput),
) ++
packageSubmissionZipSettings ++
inConfig(Grading)(Defaults.testSettings ++ Seq(
unmanagedJars += file("grading-tests.jar"),
definedTests := (definedTests in Test).value,
internalDependencyClasspath := (internalDependencyClasspath in Test).value
))
/** **********************************************************
* SUBMITTING A SOLUTION TO COURSERA
*/
val packageSubmissionZipSettings = Seq(
packageSubmissionZip := {
val submission = crossTarget.value / "submission.zip"
val sources = (packageSourcesOnly in Compile).value
val binaries = (packageBinWithoutResources in Compile).value
IO.zip(Seq(sources -> "sources.zip", binaries -> "binaries.jar"), submission, None)
submission
},
artifactClassifier in packageSourcesOnly := Some("sources"),
artifact in (Compile, packageBinWithoutResources) ~= (art => art.withName(art.name + "-without-resources"))
) ++
inConfig(Compile)(
Defaults.packageTaskSettings(packageSourcesOnly, Defaults.sourceMappings) ++
Defaults.packageTaskSettings(packageBinWithoutResources, Def.task {
val relativePaths =
(unmanagedResources in Compile).value.flatMap(Path.relativeTo((unmanagedResourceDirectories in Compile).value)(_))
(mappings in (Compile, packageBin)).value.filterNot { case (_, path) => relativePaths.contains(path) }
})
submitSetting,
)
val maxSubmitFileSize = {
val mb = 1024 * 1024
10 * mb
}
/** Check that the jar exists, isn't empty, isn't crazy big, and can be read
* If so, encode jar as base64 so we can send it to Coursera
*/
def prepareJar(jar: File, s: TaskStreams): String = {
val errPrefix = "Error submitting assignment jar: "
val fileLength = jar.length()
if (!jar.exists()) {
s.log.error(errPrefix + "jar archive does not exist\n" + jar.getAbsolutePath)
failSubmit()
} else if (fileLength == 0L) {
s.log.error(errPrefix + "jar archive is empty\n" + jar.getAbsolutePath)
failSubmit()
} else if (fileLength > maxSubmitFileSize) {
s.log.error(errPrefix + "jar archive is too big. Allowed size: " +
maxSubmitFileSize + " bytes, found " + fileLength + " bytes.\n" +
jar.getAbsolutePath)
failSubmit()
} else {
val bytes = new Array[Byte](fileLength.toInt)
val sizeRead = try {
val is = new FileInputStream(jar)
val read = is.read(bytes)
is.close()
read
} catch {
case ex: IOException =>
s.log.error(errPrefix + "failed to read sources jar archive\n" + ex.toString)
failSubmit()
}
if (sizeRead != bytes.length) {
s.log.error(errPrefix + "failed to read the sources jar archive, size read: " + sizeRead)
failSubmit()
} else encodeBase64(bytes)
}
}
/** Task to package solution to a given file path */
lazy val packageSubmissionSetting = packageSubmission := {
val args: Seq[String] = Def.spaceDelimited("[path]").parsed
val s: TaskStreams = streams.value // for logging
val jar = (packageSubmissionZip in Compile).value
val base64Jar = prepareJar(jar, s)
val path = args.headOption.getOrElse((baseDirectory.value / "submission.jar").absolutePath)
scala.tools.nsc.io.File(path).writeAll(base64Jar)
}
/*
/** Task to submit a solution to coursera */
val submit = inputKey[Unit]("submit solution to Coursera")
lazy val submitSetting = submit := {
// Fail if scalafix linting does not pass.
scalafixLinting.value
StudentTasks.scalafixLinting.value
val args: Seq[String] = Def.spaceDelimited("<arg>").parsed
val s: TaskStreams = streams.value // for logging
val jar = (packageSubmissionZip in Compile).value
val jar = (Compile / packageSubmissionZip).value
val assignmentDetails =
courseraId.?.value.getOrElse(throw new MessageOnlyException("This assignment can not be submitted to Coursera because the `courseraId` setting is undefined"))
val assignmentKey = assignmentDetails.key
val courseName =
course.value match {
case "capstone" => "scala-capstone"
case "progfun1" => "scala-functional-programming"
case "progfun2" => "scala-functional-program-design"
case "parprog1" => "scala-parallel-programming"
case "bigdata" => "scala-spark-big-data"
case "capstone" => "scala-capstone"
case "reactive" => "scala-akka-reactive"
case other => other
}
......@@ -166,10 +86,10 @@ object StudentTasks extends AutoPlugin {
}
""".stripMargin
s.log.error(inputErr)
failSubmit()
StudentTasks.failSubmit()
}
val base64Jar = prepareJar(jar, s)
val base64Jar = StudentTasks.prepareJar(jar, s)
val json =
s"""|{
| "assignmentKey":"$assignmentKey",
......@@ -288,16 +208,5 @@ object StudentTasks extends AutoPlugin {
}
}
*/
def failSubmit(): Nothing = {
sys.error("Submission failed")
}
/**
* *****************
* DEALING WITH JARS
*/
def encodeBase64(bytes: Array[Byte]): String =
new String(Base64.encodeBase64(bytes))
}
......@@ -3,47 +3,49 @@ package ch.epfl.lamp
import sbt._
import sbt.Keys._
/**
* Coursera uses two versions of each assignment. They both have the same assignment key and part id but have
* different item ids.
*
* @param key Assignment key
* @param partId Assignment partId
* @param itemId Item id of the non premium version
* @param premiumItemId Item id of the premium version (`None` if the assignment is optional)
*/
case class CourseraId(key: String, partId: String, itemId: String, premiumItemId: Option[String])
/**
* Settings shared by all assignments, reused in various tasks.
*/
object MOOCSettings extends AutoPlugin {
override def requires = super.requires && filteringReporterPlugin.FilteringReporterPlugin
object autoImport {
val course = SettingKey[String]("course")
val assignment = SettingKey[String]("assignment")
val options = SettingKey[Map[String, Map[String, String]]]("options")
val courseraId = settingKey[CourseraId]("Coursera-specific information identifying the assignment")
val testSuite = settingKey[String]("Fully qualified name of the test suite of this assignment")
.withRank(KeyRanks.Invisible)
// Convenient alias
type CourseraId = ch.epfl.lamp.CourseraId
val CourseraId = ch.epfl.lamp.CourseraId
val datasetUrl = settingKey[String]("URL of the dataset used for testing")
val downloadDataset = taskKey[File]("Download the dataset required for the assignment")
val assignmentVersion = settingKey[String]("Hash string indicating the version of the assignment")
}
import autoImport._
override val globalSettings: Seq[Def.Setting[_]] = Seq(
// supershell is verbose, buggy and useless.
useSuperShell := false
)
lazy val downloadDatasetDef = downloadDataset := {
val logger = streams.value.log
datasetUrl.?.value match {
case Some(url) =>
import scalaj.http.Http
import sbt.io.IO
val dest = (Compile / resourceManaged).value / assignment.value / url.split("/").last
if (!dest.exists()) {
IO.touch(dest)
logger.info(s"Downloading $url")
val res = Http(url).method("GET")
val is = res.asBytes.body
IO.write(dest, is)
}
dest
case None =>
logger.info(s"No dataset defined in datasetUrl")
throw new sbt.MessageOnlyException("No dataset to download for this assignment")
}
}
override val projectSettings: Seq[Def.Setting[_]] = Seq(
parallelExecution in Test := false,
downloadDatasetDef,
Test / parallelExecution := false,
// Report test result after each test instead of waiting for every test to finish
logBuffered in Test := false,
Test / logBuffered := false,
name := s"${course.value}-${assignment.value}"
)
}
......@@ -2,12 +2,10 @@ package ch.epfl.lamp
import sbt._
import Keys._
import scalafix.sbt.ScalafixPlugin.autoImport._
// import scalaj.http._
import java.io.{File, FileInputStream, IOException}
import org.apache.commons.codec.binary.Base64
// import play.api.libs.json.{Json, JsObject, JsPath}
import scala.util.{Failure, Success, Try}
/**
* Provides tasks for submitting the assignment
......@@ -19,52 +17,70 @@ object StudentTasks extends AutoPlugin {
object autoImport {
val packageSourcesOnly = TaskKey[File]("packageSourcesOnly", "Package the sources of the project")
val packageBinWithoutResources = TaskKey[File]("packageBinWithoutResources", "Like packageBin, but without the resources")
val packageSubmissionZip = TaskKey[File]("packageSubmissionZip")
val packageSubmission = inputKey[Unit]("package solution as an archive file")
val packageSubmission = inputKey[Unit]("package solution as an archive file")
lazy val Grading = config("grading") extend(Runtime)
}
import autoImport._
import MOOCSettings.autoImport._
// Run scalafix linting after compilation to avoid seeing parser errors twice
// Keep in sync with the use of scalafix in Grader
// (--exclude doesn't work (https://github.com/lampepfl-courses/moocs/pull/28#issuecomment-427894795)
// so we customize unmanagedSources below instead)
val scalafixLinting = Def.taskDyn {
if (new File(".scalafix.conf").exists()) {
(Compile / scalafix).toTask(" --check").dependsOn(Compile / compile)
} else Def.task(())
}
val testsJar = file("grading-tests.jar")
override lazy val projectSettings = Seq(
// Run scalafix linting in parallel with the tests
(Test / test) := {
scalafixLinting.value
(Test / test).value
},
packageSubmissionSetting,
fork := true,
connectInput in run := true,
run / connectInput := true,
outputStrategy := Some(StdoutOutput),
) ++
packageSubmissionZipSettings ++
inConfig(Grading)(Defaults.testSettings ++ Seq(
unmanagedJars += file("grading-tests.jar"),
definedTests := (definedTests in Test).value,
internalDependencyClasspath := (internalDependencyClasspath in Test).value
scalafixConfig := {
val scalafixDotConf = (baseDirectory.value / ".scalafix.conf")
if (scalafixDotConf.exists) Some(scalafixDotConf) else None
}
) ++ packageSubmissionZipSettings ++ (
if(testsJar.exists) inConfig(Grading)(Defaults.testSettings ++ Seq(
unmanagedJars += testsJar,
definedTests := (Test / definedTests).value,
internalDependencyClasspath := (Test / internalDependencyClasspath).value,
managedClasspath := (Test / managedClasspath).value,
))
/** **********************************************************
* SUBMITTING A SOLUTION TO COURSERA
*/
else Nil
)
val packageSubmissionZipSettings = Seq(
packageSubmissionZip := {
val submission = crossTarget.value / "submission.zip"
val sources = (packageSourcesOnly in Compile).value
val binaries = (packageBinWithoutResources in Compile).value
val sources = (Compile / packageSourcesOnly).value
val binaries = (Compile / packageBinWithoutResources).value
IO.zip(Seq(sources -> "sources.zip", binaries -> "binaries.jar"), submission, None)
submission
},
artifactClassifier in packageSourcesOnly := Some("sources"),
artifact in (Compile, packageBinWithoutResources) ~= (art => art.withName(art.name + "-without-resources"))
packageSourcesOnly / artifactClassifier := Some("sources"),
Compile / packageBinWithoutResources / artifact ~= (art => art.withName(art.name + "-without-resources"))
) ++
inConfig(Compile)(
Defaults.packageTaskSettings(packageSourcesOnly, Defaults.sourceMappings) ++
Defaults.packageTaskSettings(packageBinWithoutResources, Def.task {
val relativePaths =
(unmanagedResources in Compile).value.flatMap(Path.relativeTo((unmanagedResourceDirectories in Compile).value)(_))
(mappings in (Compile, packageBin)).value.filterNot { case (_, path) => relativePaths.contains(path) }
(Compile / resources).value.flatMap(Path.relativeTo((Compile / resourceDirectories).value)(_))
(Compile / packageBin / mappings).value.filterNot { case (_, path) => relativePaths.contains(path) }
})
)
......@@ -73,9 +89,6 @@ object StudentTasks extends AutoPlugin {
10 * mb
}
/** Check that the jar exists, isn't empty, isn't crazy big, and can be read
* If so, encode jar as base64 so we can send it to Coursera
*/
def prepareJar(jar: File, s: TaskStreams): String = {
val errPrefix = "Error submitting assignment jar: "
val fileLength = jar.length()
......@@ -111,9 +124,12 @@ object StudentTasks extends AutoPlugin {
/** Task to package solution to a given file path */
lazy val packageSubmissionSetting = packageSubmission := {
// Fail if scalafix linting does not pass.
scalafixLinting.value
val args: Seq[String] = Def.spaceDelimited("[path]").parsed
val s: TaskStreams = streams.value // for logging
val jar = (packageSubmissionZip in Compile).value
val jar = (Compile / packageSubmissionZip).value
val base64Jar = prepareJar(jar, s)
......@@ -121,175 +137,6 @@ object StudentTasks extends AutoPlugin {
scala.tools.nsc.io.File(path).writeAll(base64Jar)
}
/*
/** Task to submit a solution to coursera */
val submit = inputKey[Unit]("submit solution to Coursera")
lazy val submitSetting = submit := {
// Fail if scalafix linting does not pass.
scalafixLinting.value
val args: Seq[String] = Def.spaceDelimited("<arg>").parsed
val s: TaskStreams = streams.value // for logging
val jar = (packageSubmissionZip in Compile).value
val assignmentDetails =
courseraId.?.value.getOrElse(throw new MessageOnlyException("This assignment can not be submitted to Coursera because the `courseraId` setting is undefined"))
val assignmentKey = assignmentDetails.key
val courseName =
course.value match {
case "capstone" => "scala-capstone"
case "bigdata" => "scala-spark-big-data"
case other => other
}
val partId = assignmentDetails.partId
val itemId = assignmentDetails.itemId
val premiumItemId = assignmentDetails.premiumItemId
val (email, secret) = args match {
case email :: secret :: Nil =>
(email, secret)
case _ =>
val inputErr =
s"""|Invalid input to `submit`. The required syntax for `submit` is:
|submit <email-address> <submit-token>
|
|The submit token is NOT YOUR LOGIN PASSWORD.
|It can be obtained from the assignment page:
|https://www.coursera.org/learn/$courseName/programming/$itemId
|${
premiumItemId.fold("") { id =>
s"""or (for premium learners):
|https://www.coursera.org/learn/$courseName/programming/$id
""".stripMargin
}
}
""".stripMargin
s.log.error(inputErr)
failSubmit()
}
val base64Jar = prepareJar(jar, s)
val json =
s"""|{
| "assignmentKey":"$assignmentKey",
| "submitterEmail":"$email",
| "secret":"$secret",
| "parts":{
| "$partId":{
| "output":"$base64Jar"
| }
| }
|}""".stripMargin
def postSubmission[T](data: String): Try[HttpResponse[String]] = {
val http = Http("https://www.coursera.org/api/onDemandProgrammingScriptSubmissions.v1")
val hs = List(
("Cache-Control", "no-cache"),
("Content-Type", "application/json")
)
s.log.info("Connecting to Coursera...")
val response = Try(http.postData(data)
.headers(hs)
.option(HttpOptions.connTimeout(10000)) // scalaj default timeout is only 100ms, changing that to 10s
.asString) // kick off HTTP POST
response
}
val connectMsg =
s"""|Attempting to submit "${assignment.value}" assignment in "$courseName" course
|Using:
|- email: $email
|- submit token: $secret""".stripMargin
s.log.info(connectMsg)
def reportCourseraResponse(response: HttpResponse[String]): Unit = {
val code = response.code
val respBody = response.body
/* Sample JSON response from Coursera
{
"message": "Invalid email or token.",
"details": {
"learnerMessage": "Invalid email or token."
}
}
*/
// Success, Coursera responds with 2xx HTTP status code
if (response.is2xx) {
val successfulSubmitMsg =
s"""|Successfully connected to Coursera. (Status $code)
|
|Assignment submitted successfully!
|
|You can see how you scored by going to:
|https://www.coursera.org/learn/$courseName/programming/$itemId/
|${
premiumItemId.fold("") { id =>
s"""or (for premium learners):
|https://www.coursera.org/learn/$courseName/programming/$id
""".stripMargin
}
}
|and clicking on "My Submission".""".stripMargin
s.log.info(successfulSubmitMsg)
}
// Failure, Coursera responds with 4xx HTTP status code (client-side failure)
else if (response.is4xx) {
val result = Try(Json.parse(respBody)).toOption
val learnerMsg = result match {
case Some(resp: JsObject) =>
(JsPath \ "details" \ "learnerMessage").read[String].reads(resp).get
case Some(x) => // shouldn't happen
"Could not parse Coursera's response:\n" + x
case None =>
"Could not parse Coursera's response:\n" + respBody
}
val failedSubmitMsg =
s"""|Submission failed.
|There was something wrong while attempting to submit.
|Coursera says:
|$learnerMsg (Status $code)""".stripMargin
s.log.error(failedSubmitMsg)
}
// Failure, Coursera responds with 5xx HTTP status code (server-side failure)
else if (response.is5xx) {
val failedSubmitMsg =
s"""|Submission failed.
|Coursera seems to be unavailable at the moment (Status $code)
|Check https://status.coursera.org/ and try again in a few minutes.
""".stripMargin
s.log.error(failedSubmitMsg)
}
// Failure, Coursera repsonds with an unexpected status code
else {
val failedSubmitMsg =
s"""|Submission failed.
|Coursera replied with an unexpected code (Status $code)
""".stripMargin
s.log.error(failedSubmitMsg)
}
}
// kick it all off, actually make request
postSubmission(json) match {
case Success(resp) => reportCourseraResponse(resp)
case Failure(e) =>
val failedConnectMsg =
s"""|Connection to Coursera failed.
|There was something wrong while attempting to connect to Coursera.
|Check your internet connection.
|${e.toString}""".stripMargin
s.log.error(failedConnectMsg)
}
}
*/
def failSubmit(): Nothing = {
sys.error("Submission failed")
}
......
sbt.version=1.6.1
// Used for Coursera submission (StudentPlugin)
libraryDependencies += "org.scalaj" %% "scalaj-http" % "2.4.2"
libraryDependencies += "com.typesafe.play" %% "play-json" % "2.9.2"
// Used for Base64 (StudentPlugin)
libraryDependencies += "commons-codec" % "commons-codec" % "1.15"
\ No newline at end of file
// addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.28")
addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.9.26")
addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.8")
addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % "0.5.3")
play.application.loader=concpar21final01.MyApplicationLoader
package f1
package concpar21final01
import play.api.{ApplicationLoader, BuiltInComponentsFromContext}
import play.api.mvc.Results.Ok
import play.api.routing.sird._
import play.api.routing.sird.*
import play.api.routing.Router
import play.api.ApplicationLoader.Context
import play.filters.HttpFiltersComponents
......@@ -11,20 +11,21 @@ import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.Future
import scala.util.Random
class MyApplicationLoader extends ApplicationLoader {
class MyApplicationLoader extends ApplicationLoader:
def load(context: Context) =
new MyComponents(context).application
}
class MyComponents(context: Context)
extends BuiltInComponentsFromContext(context)
with HttpFiltersComponents {
with HttpFiltersComponents:
lazy val router = Router.from {
case GET(p"/") =>
Action.async {
(new F1MockData).leaderboard().map(leaderboardHTML).map(Ok(_).as("text/html"))
}
lazy val router = Router.from { case GET(p"/") =>
Action.async {
(new Problem1MockData)
.leaderboard()
.map(leaderboardHTML)
.map(Ok(_).as("text/html"))
}
}
def leaderboardHTML(data: List[Grade]): String =
......@@ -37,17 +38,18 @@ class MyComponents(context: Context)
| <body>
| <h1>Leaderboard:</h1>
| <ul>
| ${data.map { case Grade(sciper, g) =>
val grade = "%1.2f".format(g)
s"<li>$sciper : $grade</li>"
}.mkString("\n ")}
| ${data
.map { case Grade(sciper, g) =>
val grade = "%1.2f".format(g)
s"<li>$sciper : $grade</li>"
}
.mkString("\n ")}
| </ul>
| </body>
|</html>
""".trim.stripMargin
}
class F1MockData extends F1 {
class Problem1MockData extends Problem1:
def getGrade(sciper: Int): Future[Option[Grade]] =
Future {
// In an actual implementation, this is where we would make a call to
......@@ -55,13 +57,12 @@ class F1MockData extends F1 {
Thread.sleep(15) // GitLab is pretty fast today...
val rand = new Random(sciper)
val grade = rand.nextInt(6).toDouble + rand.nextDouble()
if (sciper < 100000 || sciper > 999999 || sciper % 10 == 0) None
if sciper < 100000 || sciper > 999999 || sciper % 10 == 0 then None
else Some(Grade(sciper, grade))
}
/**
* Retrieve the list of enrolled students from IS-academia
*/
/** Retrieve the list of enrolled students from IS-academia
*/
def getScipers(): Future[List[Int]] =
Future {
Thread.sleep(100)
......@@ -89,4 +90,3 @@ class F1MockData extends F1 {
311803, 299981, 311832, 301088, 259649, 279183, 341760, 311844, 279079,
390997, 311917, 390999, 361122, 301208, 311538, 272943, 361570, 390959)
}
}
package concpar21final01
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.Future
case class Grade(sciper: Int, grade: Double)
trait Problem1:
/** Retrieve the list of student grades, sorted such that maximum grades
* appear at the head of the list.
*/
def leaderboard(): Future[List[Grade]] =
getScipers()
.flatMap { scipers =>
Future.sequence(scipers.map(getGrade))
}
.map(_.flatten.sortBy(_.grade).reverse)
/** Retrieve a student's grade using GitLab's API. The result is wrapped in an
* option, where `Future(None)` indicates either:
* - the student is not registered to the class
* - the student did not push his/her solution to GitLab
*/
def getGrade(sciper: Int): Future[Option[Grade]]
/** Retrieve the list of enrolled students from IS-academia
*/
def getScipers(): Future[List[Int]]
package f1
package concpar21final01
import play.api.test._
import play.api.test.Helpers._
import scala.concurrent.duration._
import play.api.test.*
import play.api.test.Helpers.*
import scala.concurrent.duration.*
import scala.concurrent.Future
import scala.concurrent.ExecutionContext.Implicits.global
class F1Suite extends munit.FunSuite {
test("Retrieves grades at the end of the exam (everyone pushed something) (10pts)") {
class F1Done extends F1 {
class Problem1Suite extends munit.FunSuite:
test(
"Retrieves grades at the end of the exam (everyone pushed something) (10pts)"
) {
class Problem1Done extends Problem1:
override def getGrade(sciper: Int): Future[Option[Grade]] =
Future {
Thread.sleep(100)
......@@ -19,22 +21,21 @@ class F1Suite extends munit.FunSuite {
Thread.sleep(100)
List(1, 2, 3, 4)
}
}
val expected: List[Grade] =
List(Grade(1, 1.0), Grade(2, 2.0), Grade(3, 3.0), Grade(4, 4.0))
(new F1Done).leaderboard().map { grades =>
(new Problem1Done).leaderboard().map { grades =>
assertEquals(grades.toSet, expected.toSet)
}
}
test("Retrieves grades mid exam (some students didn't push yet) (10pts)") {
class F1Partial extends F1 {
class Problem1Partial extends Problem1:
override def getGrade(sciper: Int): Future[Option[Grade]] =
Future {
Thread.sleep(100)
if (sciper % 2 == 0) None
if sciper % 2 == 0 then None
else Some(Grade(sciper, sciper))
}
override def getScipers(): Future[List[Int]] =
......@@ -42,18 +43,17 @@ class F1Suite extends munit.FunSuite {
Thread.sleep(100)
List(1, 2, 3, 4)
}
}
val expected: List[Grade] =
List(Grade(1, 1.0), Grade(3, 3.0))
(new F1Partial).leaderboard().map { grades =>
(new Problem1Partial).leaderboard().map { grades =>
assertEquals(grades.toSet, expected.toSet)
}
}
test("The output list is sorted by grade (10pts)") {
(new F1MockData).leaderboard().map { grades =>
(new Problem1MockData).leaderboard().map { grades =>
assert(grades.size >= 176)
assert(grades.zipWithIndex.forall { case (g, i) =>
grades.drop(i).forall(x => g.grade >= x.grade)
......@@ -64,21 +64,19 @@ class F1Suite extends munit.FunSuite {
test("GitLab API calls are done in parallel (2pts)") {
var inParallel: Boolean = false
class F1Par extends F1MockData {
class Problem1Par extends Problem1MockData:
var in: Boolean = false
override def getGrade(sciper: Int): Future[Option[Grade]] = {
override def getGrade(sciper: Int): Future[Option[Grade]] =
Future {
if (in) inParallel = true
if in then inParallel = true
in = true
val out = super.getGrade(sciper)
in = false
concurrent.Await.result(out, Duration(10, SECONDS))
}
}
}
(new F1Par).leaderboard().map { grades =>
(new Problem1Par).leaderboard().map { grades =>
assert(grades.size >= 176)
assert(inParallel)
}
......@@ -87,16 +85,13 @@ class F1Suite extends munit.FunSuite {
test("The IS-academia API is called exactly once (2pts)") {
var called: Int = 0
class F1Once extends F1MockData {
override def getScipers(): Future[List[Int]] = {
class Problem1Once extends Problem1MockData:
override def getScipers(): Future[List[Int]] =
called += 1
super.getScipers()
}
}
(new F1Once).leaderboard().map { grades =>
(new Problem1Once).leaderboard().map { grades =>
assert(grades.size >= 176)
assert(called == 1)
}
}
}
# General
*.DS_Store
*.swp
*~
# Dotty
*.class
*.tasty
*.hasTasty
# sbt
target/
# IDE
.bsp
logs/
.bloop
.bsp
.dotty-ide-artifact
.dotty-ide.json
.idea
.metals
.vscode
# datasets
stackoverflow-grading.csv
wikipedia-grading.dat
*.csv
*.dat
metals.sbt
// Student tasks (i.e. submit, packageSubmission)
enablePlugins(StudentTasks)
assignmentVersion.withRank(KeyRanks.Invisible) := "eadbf7a6"
course := "final"
assignment := "f4"
course := "concpar"
assignment := "concpar21final02"
scalaVersion := "3.1.0"
scalaVersion := "3.0.0-RC1"
scalacOptions ++= Seq("-language:implicitConversions", "-deprecation")
libraryDependencies += "org.scalameta" %% "munit" % "1.0.0-M3" % Test
val akkaVersion = "2.6.19"
val logbackVersion = "1.2.11"
libraryDependencies ++= Seq(
("org.apache.spark" %% "spark-core" % "3.2.0-SNAPSHOT").withDottyCompat(scalaVersion.value),
"com.typesafe.akka" %% "akka-actor" % akkaVersion,
"com.typesafe.akka" %% "akka-testkit" % akkaVersion,
// SLF4J backend
// See https://doc.akka.io/docs/akka/current/typed/logging.html#slf4j-backend
"ch.qos.logback" % "logback-classic" % logbackVersion
)
// Contains Spark 3 snapshot built against 2.13: https://github.com/smarter/spark/tree/scala-2.13
resolvers += "Spark Snapshots Copy" at "https://scala-webapps.epfl.ch/artifactory/spark-snapshot/"
libraryDependencies += "org.scalameta" %% "munit" % "0.7.22"
fork := true
javaOptions ++= Seq("-Dakka.loglevel=Error", "-Dakka.actor.debug.receive=on")
val MUnitFramework = new TestFramework("munit.Framework")
testFrameworks += MUnitFramework
// Decode Scala names
testOptions += Tests.Argument(MUnitFramework, "-s")
testSuite := "f4.F4Suite"
// Without forking, ctrl-c doesn't actually fully stop Spark
fork in run := true
fork in Test := true
......@@ -2,143 +2,63 @@ package ch.epfl.lamp
import sbt._
import Keys._
// import scalaj.http._
import java.io.{File, FileInputStream, IOException}
import org.apache.commons.codec.binary.Base64
// import play.api.libs.json.{Json, JsObject, JsPath}
import scala.util.{Failure, Success, Try}
import scalaj.http._
import play.api.libs.json.{Json, JsObject, JsPath}
/**
* Provides tasks for submitting the assignment
*/
object StudentTasks extends AutoPlugin {
override def requires = super.requires && MOOCSettings
* Coursera uses two versions of each assignment. They both have the same assignment key and part id but have
* different item ids.
*
* @param key Assignment key
* @param partId Assignment partId
* @param itemId Item id of the non premium version
* @param premiumItemId Item id of the premium version (`None` if the assignment is optional)
*/
case class CourseraId(courseId: String, key: String, partId: String, itemId: String, premiumItemId: Option[String])
object autoImport {
val packageSourcesOnly = TaskKey[File]("packageSourcesOnly", "Package the sources of the project")
val packageBinWithoutResources = TaskKey[File]("packageBinWithoutResources", "Like packageBin, but without the resources")
val packageSubmissionZip = TaskKey[File]("packageSubmissionZip")
val packageSubmission = inputKey[Unit]("package solution as an archive file")
lazy val Grading = config("grading") extend(Runtime)
}
object CourseraStudent extends AutoPlugin {
override def requires = super.requires && MOOCSettings
import autoImport._
object autoImport {
val options = SettingKey[Map[String, Map[String, String]]]("options")
val courseraId = settingKey[CourseraId]("Coursera-specific information identifying the assignment")
// Convenient alias
type CourseraId = ch.epfl.lamp.CourseraId
val CourseraId = ch.epfl.lamp.CourseraId
}
import StudentTasks.autoImport._
import MOOCSettings.autoImport._
import autoImport._
override lazy val projectSettings = Seq(
packageSubmissionSetting,
fork := true,
connectInput in run := true,
outputStrategy := Some(StdoutOutput),
) ++
packageSubmissionZipSettings ++
inConfig(Grading)(Defaults.testSettings ++ Seq(
unmanagedJars += file("grading-tests.jar"),
definedTests := (definedTests in Test).value,
internalDependencyClasspath := (internalDependencyClasspath in Test).value
))
/** **********************************************************
* SUBMITTING A SOLUTION TO COURSERA
*/
val packageSubmissionZipSettings = Seq(
packageSubmissionZip := {
val submission = crossTarget.value / "submission.zip"
val sources = (packageSourcesOnly in Compile).value
val binaries = (packageBinWithoutResources in Compile).value
IO.zip(Seq(sources -> "sources.zip", binaries -> "binaries.jar"), submission, None)
submission
},
artifactClassifier in packageSourcesOnly := Some("sources"),
artifact in (Compile, packageBinWithoutResources) ~= (art => art.withName(art.name + "-without-resources"))
) ++
inConfig(Compile)(
Defaults.packageTaskSettings(packageSourcesOnly, Defaults.sourceMappings) ++
Defaults.packageTaskSettings(packageBinWithoutResources, Def.task {
val relativePaths =
(unmanagedResources in Compile).value.flatMap(Path.relativeTo((unmanagedResourceDirectories in Compile).value)(_))
(mappings in (Compile, packageBin)).value.filterNot { case (_, path) => relativePaths.contains(path) }
})
submitSetting,
)
val maxSubmitFileSize = {
val mb = 1024 * 1024
10 * mb
}
/** Check that the jar exists, isn't empty, isn't crazy big, and can be read
* If so, encode jar as base64 so we can send it to Coursera
*/
def prepareJar(jar: File, s: TaskStreams): String = {
val errPrefix = "Error submitting assignment jar: "
val fileLength = jar.length()
if (!jar.exists()) {
s.log.error(errPrefix + "jar archive does not exist\n" + jar.getAbsolutePath)
failSubmit()
} else if (fileLength == 0L) {
s.log.error(errPrefix + "jar archive is empty\n" + jar.getAbsolutePath)
failSubmit()
} else if (fileLength > maxSubmitFileSize) {
s.log.error(errPrefix + "jar archive is too big. Allowed size: " +
maxSubmitFileSize + " bytes, found " + fileLength + " bytes.\n" +
jar.getAbsolutePath)
failSubmit()
} else {
val bytes = new Array[Byte](fileLength.toInt)
val sizeRead = try {
val is = new FileInputStream(jar)
val read = is.read(bytes)
is.close()
read
} catch {
case ex: IOException =>
s.log.error(errPrefix + "failed to read sources jar archive\n" + ex.toString)
failSubmit()
}
if (sizeRead != bytes.length) {
s.log.error(errPrefix + "failed to read the sources jar archive, size read: " + sizeRead)
failSubmit()
} else encodeBase64(bytes)
}
}
/** Task to package solution to a given file path */
lazy val packageSubmissionSetting = packageSubmission := {
val args: Seq[String] = Def.spaceDelimited("[path]").parsed
val s: TaskStreams = streams.value // for logging
val jar = (packageSubmissionZip in Compile).value
val base64Jar = prepareJar(jar, s)
val path = args.headOption.getOrElse((baseDirectory.value / "submission.jar").absolutePath)
scala.tools.nsc.io.File(path).writeAll(base64Jar)
}
/*
/** Task to submit a solution to coursera */
val submit = inputKey[Unit]("submit solution to Coursera")
lazy val submitSetting = submit := {
// Fail if scalafix linting does not pass.
scalafixLinting.value
StudentTasks.scalafixLinting.value
val args: Seq[String] = Def.spaceDelimited("<arg>").parsed
val s: TaskStreams = streams.value // for logging
val jar = (packageSubmissionZip in Compile).value
val jar = (Compile / packageSubmissionZip).value
val assignmentDetails =
courseraId.?.value.getOrElse(throw new MessageOnlyException("This assignment can not be submitted to Coursera because the `courseraId` setting is undefined"))
val assignmentKey = assignmentDetails.key
val courseName =
course.value match {
case "capstone" => "scala-capstone"
case "progfun1" => "scala-functional-programming"
case "progfun2" => "scala-functional-program-design"
case "parprog1" => "scala-parallel-programming"
case "bigdata" => "scala-spark-big-data"
case "capstone" => "scala-capstone"
case "reactive" => "scala-akka-reactive"
case other => other
}
......@@ -166,10 +86,10 @@ object StudentTasks extends AutoPlugin {
}
""".stripMargin
s.log.error(inputErr)
failSubmit()
StudentTasks.failSubmit()
}
val base64Jar = prepareJar(jar, s)
val base64Jar = StudentTasks.prepareJar(jar, s)
val json =
s"""|{
| "assignmentKey":"$assignmentKey",
......@@ -288,16 +208,5 @@ object StudentTasks extends AutoPlugin {
}
}
*/
def failSubmit(): Nothing = {
sys.error("Submission failed")
}
/**
* *****************
* DEALING WITH JARS
*/
def encodeBase64(bytes: Array[Byte]): String =
new String(Base64.encodeBase64(bytes))
}
......@@ -3,47 +3,49 @@ package ch.epfl.lamp
import sbt._
import sbt.Keys._
/**
* Coursera uses two versions of each assignment. They both have the same assignment key and part id but have
* different item ids.
*
* @param key Assignment key
* @param partId Assignment partId
* @param itemId Item id of the non premium version
* @param premiumItemId Item id of the premium version (`None` if the assignment is optional)
*/
case class CourseraId(key: String, partId: String, itemId: String, premiumItemId: Option[String])
/**
* Settings shared by all assignments, reused in various tasks.
*/
object MOOCSettings extends AutoPlugin {
override def requires = super.requires && filteringReporterPlugin.FilteringReporterPlugin
object autoImport {
val course = SettingKey[String]("course")
val assignment = SettingKey[String]("assignment")
val options = SettingKey[Map[String, Map[String, String]]]("options")
val courseraId = settingKey[CourseraId]("Coursera-specific information identifying the assignment")
val testSuite = settingKey[String]("Fully qualified name of the test suite of this assignment")
.withRank(KeyRanks.Invisible)
// Convenient alias
type CourseraId = ch.epfl.lamp.CourseraId
val CourseraId = ch.epfl.lamp.CourseraId
val datasetUrl = settingKey[String]("URL of the dataset used for testing")
val downloadDataset = taskKey[File]("Download the dataset required for the assignment")
val assignmentVersion = settingKey[String]("Hash string indicating the version of the assignment")
}
import autoImport._
override val globalSettings: Seq[Def.Setting[_]] = Seq(
// supershell is verbose, buggy and useless.
useSuperShell := false
)
lazy val downloadDatasetDef = downloadDataset := {
val logger = streams.value.log
datasetUrl.?.value match {
case Some(url) =>
import scalaj.http.Http
import sbt.io.IO
val dest = (Compile / resourceManaged).value / assignment.value / url.split("/").last
if (!dest.exists()) {
IO.touch(dest)
logger.info(s"Downloading $url")
val res = Http(url).method("GET")
val is = res.asBytes.body
IO.write(dest, is)
}
dest
case None =>
logger.info(s"No dataset defined in datasetUrl")
throw new sbt.MessageOnlyException("No dataset to download for this assignment")
}
}
override val projectSettings: Seq[Def.Setting[_]] = Seq(
parallelExecution in Test := false,
downloadDatasetDef,
Test / parallelExecution := false,
// Report test result after each test instead of waiting for every test to finish
logBuffered in Test := false,
Test / logBuffered := false,
name := s"${course.value}-${assignment.value}"
)
}
package ch.epfl.lamp
import sbt._
import Keys._
import scalafix.sbt.ScalafixPlugin.autoImport._
import java.io.{File, FileInputStream, IOException}
import org.apache.commons.codec.binary.Base64
/**
* Provides tasks for submitting the assignment
*/
object StudentTasks extends AutoPlugin {
override def requires = super.requires && MOOCSettings
object autoImport {
val packageSourcesOnly = TaskKey[File]("packageSourcesOnly", "Package the sources of the project")
val packageBinWithoutResources = TaskKey[File]("packageBinWithoutResources", "Like packageBin, but without the resources")
val packageSubmissionZip = TaskKey[File]("packageSubmissionZip")
val packageSubmission = inputKey[Unit]("package solution as an archive file")
lazy val Grading = config("grading") extend(Runtime)
}
import autoImport._
// Run scalafix linting after compilation to avoid seeing parser errors twice
// Keep in sync with the use of scalafix in Grader
// (--exclude doesn't work (https://github.com/lampepfl-courses/moocs/pull/28#issuecomment-427894795)
// so we customize unmanagedSources below instead)
val scalafixLinting = Def.taskDyn {
if (new File(".scalafix.conf").exists()) {
(Compile / scalafix).toTask(" --check").dependsOn(Compile / compile)
} else Def.task(())
}
val testsJar = file("grading-tests.jar")
override lazy val projectSettings = Seq(
// Run scalafix linting in parallel with the tests
(Test / test) := {
scalafixLinting.value
(Test / test).value
},
packageSubmissionSetting,
fork := true,
run / connectInput := true,
outputStrategy := Some(StdoutOutput),
scalafixConfig := {
val scalafixDotConf = (baseDirectory.value / ".scalafix.conf")
if (scalafixDotConf.exists) Some(scalafixDotConf) else None
}
) ++ packageSubmissionZipSettings ++ (
if(testsJar.exists) inConfig(Grading)(Defaults.testSettings ++ Seq(
unmanagedJars += testsJar,
definedTests := (Test / definedTests).value,
internalDependencyClasspath := (Test / internalDependencyClasspath).value,
managedClasspath := (Test / managedClasspath).value,
))
else Nil
)
val packageSubmissionZipSettings = Seq(
packageSubmissionZip := {
val submission = crossTarget.value / "submission.zip"
val sources = (Compile / packageSourcesOnly).value
val binaries = (Compile / packageBinWithoutResources).value
IO.zip(Seq(sources -> "sources.zip", binaries -> "binaries.jar"), submission, None)
submission
},
packageSourcesOnly / artifactClassifier := Some("sources"),
Compile / packageBinWithoutResources / artifact ~= (art => art.withName(art.name + "-without-resources"))
) ++
inConfig(Compile)(
Defaults.packageTaskSettings(packageSourcesOnly, Defaults.sourceMappings) ++
Defaults.packageTaskSettings(packageBinWithoutResources, Def.task {
val relativePaths =
(Compile / resources).value.flatMap(Path.relativeTo((Compile / resourceDirectories).value)(_))
(Compile / packageBin / mappings).value.filterNot { case (_, path) => relativePaths.contains(path) }
})
)
val maxSubmitFileSize = {
val mb = 1024 * 1024
10 * mb
}
def prepareJar(jar: File, s: TaskStreams): String = {
val errPrefix = "Error submitting assignment jar: "
val fileLength = jar.length()
if (!jar.exists()) {
s.log.error(errPrefix + "jar archive does not exist\n" + jar.getAbsolutePath)
failSubmit()
} else if (fileLength == 0L) {
s.log.error(errPrefix + "jar archive is empty\n" + jar.getAbsolutePath)
failSubmit()
} else if (fileLength > maxSubmitFileSize) {
s.log.error(errPrefix + "jar archive is too big. Allowed size: " +
maxSubmitFileSize + " bytes, found " + fileLength + " bytes.\n" +
jar.getAbsolutePath)
failSubmit()
} else {
val bytes = new Array[Byte](fileLength.toInt)
val sizeRead = try {
val is = new FileInputStream(jar)
val read = is.read(bytes)
is.close()
read
} catch {
case ex: IOException =>
s.log.error(errPrefix + "failed to read sources jar archive\n" + ex.toString)
failSubmit()
}
if (sizeRead != bytes.length) {
s.log.error(errPrefix + "failed to read the sources jar archive, size read: " + sizeRead)
failSubmit()
} else encodeBase64(bytes)
}
}
/** Task to package solution to a given file path */
lazy val packageSubmissionSetting = packageSubmission := {
// Fail if scalafix linting does not pass.
scalafixLinting.value
val args: Seq[String] = Def.spaceDelimited("[path]").parsed
val s: TaskStreams = streams.value // for logging
val jar = (Compile / packageSubmissionZip).value
val base64Jar = prepareJar(jar, s)
val path = args.headOption.getOrElse((baseDirectory.value / "submission.jar").absolutePath)
scala.tools.nsc.io.File(path).writeAll(base64Jar)
}
def failSubmit(): Nothing = {
sys.error("Submission failed")
}
/**
* *****************
* DEALING WITH JARS
*/
def encodeBase64(bytes: Array[Byte]): String =
new String(Base64.encodeBase64(bytes))
}
sbt.version=1.6.1
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment