pythonlib: publish modules

example/package.mill

@@ -67,6 +67,7 @@ object `package` extends RootModule with Module {
   object pythonlib extends Module {
     object basic extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "basic"))
     object dependencies extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "dependencies"))
+    object publishing extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "publishing"))
   }
 
   object cli extends Module{

example/pythonlib/publishing/1-publish-module/LICENSE

@@ -0,0 +1 @@
+Public domain

example/pythonlib/publishing/1-publish-module/README.md

@@ -0,0 +1 @@
+This is an example package.

example/pythonlib/publishing/1-publish-module/build.mill

@@ -0,0 +1,17 @@
+import mill._, pythonlib._
+
+object `package` extends RootModule with PythonModule with PublishModule {
+  import PublishModule._
+
+  def pythonDeps = Seq("jinja2==3.1.4")
+
+  def publishMeta = PublishMeta(
+    name = "testpkg",
+    description = "an example package",
+    requiresPython = ">= 3.12",
+    authors = Seq(Developer("John Doe", "[email protected]"))
+  )
+
+  def publishVersion = "0.0.1"
+
+}

pythonlib/src/mill/pythonlib/PublishModule.scala

@@ -0,0 +1,237 @@
+package mill.pythonlib
+
+import mill.api.Result
+import mill._
+
+/**
+ * A python module which also defines how to build and publish source distributions and wheels.
+ */
+trait PublishModule extends PythonModule {
+  import PublishModule.PublishMeta
+
+  override def moduleDeps: Seq[PublishModule] = super.moduleDeps.map {
+    case m: PublishModule => m
+    case other =>
+      throw new Exception(
+        s"PublishModule moduleDeps need to be also PublishModules. $other is not a PublishModule"
+      )
+  }
+
+  override def pythonToolDeps = Task {
+    super.pythonToolDeps() ++ Seq("setuptools", "build", "twine")
+  }
+
+  /**
+   * Metadata about your project, required to build and publish.
+   *
+   * This is roughly equivalent to what you'd find in the general section of a `pyproject.toml` file
+   * https://packaging.python.org/en/latest/guides/writing-pyproject-toml/#about-your-project.
+   */
+  def publishMeta: T[PublishMeta]
+
+  /**
+   * The artifact version that this module would be published as.
+   */
+  def publishVersion: T[String]
+
+  /**
+   * The PEP-518-compliant `pyproject.toml` file, which describes how to package this module into a
+   * distribution (sdist and wheel).
+   *
+   * By default, Mill will generate this file for you from the information it knows (e.g.
+   * dependencies declared in [[pythonDeps]]). It will use `setuptools` as the build backend, and
+   * `build` as the frontend.
+   *
+   * You can however override this task to point to your own `pyproject.toml` file, if you need to.
+   * In this case, please note the following:
+   *
+   * - Mill will create a source distribution first, and then use that to build a binary
+   *   distribution (aka wheel). Going through this intermediary step, rather than building a wheel
+   *   directly, ensures that end users can rebuild wheels on their systems, for example if a
+   *   platform-dependent wheel is not available pre-made.
+   *
+   * - Hence, the source distribution will need to be "self contained". In particular this means
+   *   that you can't reference files by absolute path within it.
+   *
+   * - Mill creates a "staging" directory in the [[sdist]] task, which will be used to bundle
+   *   everything up into an sdist (via the `build` python command, although this is an
+   *   implementation detail). You can include additional files in this directory via the
+   *   [[buildFiles]] task.
+   */
+  def pyproject: T[PathRef] = Task {
+    val moduleNames = Task.traverse(moduleDeps)(_.publishMeta)().map(_.name)
+    val moduleVersions = Task.traverse(moduleDeps)(_.publishVersion)()
+    val moduleRequires = moduleNames.zip(moduleVersions).map { case (n, v) => s"$n==$v" }
+    val deps = (moduleRequires ++ pythonDeps()).map(s => s"\"$s\"").mkString(", ")
+
+    val str =
+      s"""|[project]
+          |name="${publishMeta().name}"
+          |version="${publishVersion()}"
+          |description="${publishMeta().description}"
+          |readme="${publishReadme().path.last}"
+          |dependencies=[${deps}]
+          |requires-python="${publishMeta().requiresPython}"
+          |
+          |[build-system]
+          |requires=["setuptools"]
+          |build-backend="setuptools.build_meta"
+          |""".stripMargin
+
+    os.write(T.dest / "pyproject.toml", str)
+    PathRef(T.dest / "pyproject.toml")
+  }
+
+  /**
+   * Files to be included in the directory used during the packaging process, apart from
+   * [[pyproject]].
+   *
+   * The format is `<destination path> -> <source path>`. Where `<destination path>` is relative to
+   * some build directory, where you'll also find `src` and `pyproject.toml`.
+   *
+   * @see [[pyproject]]
+   */
+  def buildFiles: T[Map[String, PathRef]] = Task {
+    Map(
+      publishReadme().path.last -> publishReadme()
+    )
+  }
+
+  /**
+   * The readme file to include in the published distribution.
+   */
+  def publishReadme: T[PathRef] = Task.Input {
+    os.list(millSourcePath).find(_.last.toLowerCase().startsWith("readme")) match {
+      case None =>
+        Result.Failure(
+          s"No readme file found in `${millSourcePath}`. A readme file is required for publishing distributions."
+        )
+      case Some(path) =>
+        Result.Success(PathRef(path))
+    }
+  }
+
+  /**
+   * The license file to include in the published distribution.
+   *
+   * TODO: remove this in favor of using a [SPDX license
+   * expression](https://packaging.python.org/en/latest/specifications/core-metadata/#license-expression).
+   * See this discussion https://github.com/com-lihaoyi/mill/discussions/4041 for how this can be implemented.
+   */
+  def publishLicense: T[PathRef] = Task.Input {
+    os.list(millSourcePath).find(_.last.toLowerCase().startsWith("license")) match {
+      case None =>
+        Result.Failure(
+          s"No license file found in `${millSourcePath}`. A license file is required for publishing distributions."
+        )
+      case Some(path) =>
+        Result.Success(PathRef(path))
+    }
+  }
+
+  /**
+   * Bundle everything up into a source distribution (sdist).
+   *
+   * @see [[pyproject]]
+   */
+  def sdist: T[PathRef] = Task {
+
+    // we use setup tools by default, which can only work with a single source directory, hence we
+    // flatten all source directories into a single hierarchy
+    val flattenedSrc = T.dest / "src"
+    for (dir <- sources()) {
+      for (path <- os.list(dir.path)) {
+        os.copy.into(path, flattenedSrc, mergeFolders = true, createFolders = true)
+      }
+    }
+
+    // copy over other, non-source files
+    os.copy(pyproject().path, T.dest / "pyproject.toml")
+    for ((dest, src) <- buildFiles()) {
+      os.copy(src.path, T.dest / os.SubPath(dest), createFolders = true, replaceExisting = true)
+    }
+
+    // we already do the isolation with mill
+    runner().run(("-m", "build", "--no-isolation", "--sdist"), workingDir = T.dest)
+    PathRef(os.list(T.dest / "dist").head)
+  }
+
+  /**
+   * Build a binary distribution of this module.
+   *
+   * @see [[pyproject]]
+   */
+  def wheel: T[PathRef] = Task {
+    val buildDir = T.dest / "extracted"
+
+    os.makeDir(buildDir)
+    os.call(
+      ("tar", "xf", sdist().path, "-C", buildDir),
+      cwd = T.dest
+    )
+    runner().run(
+      (
+        // format: off
+        "-m", "build",
+        "--no-isolation", // we already do the isolation with mill
+        "--wheel",
+        "--outdir", T.dest / "dist"
+        // format: on
+      ),
+      workingDir = os.list(buildDir).head // sdist archive contains a directory
+    )
+    PathRef(os.list(T.dest / "dist").head)
+  }
+
+  def repositoryUrl: T[String] = Task {
+    "https://test.pypi.org/legacy/"
+  }
+
+  def publish(
+      repositoryUrl: String = null,
+      username: String = null,
+      password: String = null
+  ): Command[Unit] = Task.Command {
+    runner().run(
+      (
+        // format: off
+        "-m", "twine",
+        "upload",
+        "--non-interactive",
+        "--repository-url", Option(repositoryUrl).getOrElse(this.repositoryUrl()),
+        Option(username).toSeq.flatMap(u => Seq("--username", u)),
+        Option(password).toSeq.flatMap(p => Seq("--password", p)),
+        sdist().path, wheel().path
+        // format: on
+      )
+    )
+  }
+
+}
+
+object PublishModule {
+
+  // https://packaging.python.org/en/latest/specifications/core-metadata/
+  case class PublishMeta(
+      name: String,
+      description: String,
+      requiresPython: String,
+      authors: Seq[Developer],
+      keywords: Seq[String] = Seq(),
+      classifiers: Seq[String] = Seq(),
+      urls: Map[String, String] = Map()
+  )
+  // https://packaging.python.org/en/latest/specifications/well-known-project-urls/#well-known-labels
+  object PublishMeta {
+    implicit val rw: upickle.default.ReadWriter[PublishMeta] = upickle.default.macroRW
+  }
+
+  case class Developer(
+      name: String,
+      email: String
+  )
+  object Developer {
+    implicit val rw: upickle.default.ReadWriter[Developer] = upickle.default.macroRW
+  }
+
+}