Encryption part 1: Restructure SDK modules (#276)

* Start moving code into common

* Share common logic in test

* Well, compiling works

* Fixing some tests

* Update readmes

* Reformat

* Fix supabase test

* Fix watchos tests

* Fix example

* Add changelog entries

* Be more explicit about potential breakage

* Review feedback

Author: simolus3

CHANGELOG.md

@@ -1,6 +1,14 @@
 # Changelog
 
-## 1.7.0 (unreleased)
+## 1.8.0 (unreleased)
+
+- Refactor SDK: `com.powersync:powersync-core` has an identical API, but now depends on 
+  `com.powersync:powersync-common` where most logic is implemented.
+  - __POTENTIALLY BREAKING CHANGE__: If you were injecting a `DatabaseDriverFactory` into Koin or Dagger, note that the
+    `PowerSyncDatabase()` factory method now takes a more generic `PersistentConnectionFactory`.
+  - If you're using `PowerSyncDatabase.inMemory`, you explicitly have to import `com.powersync.inMemory` now.
+
+## 1.7.0
 
 - Add `PowerSyncDatabase.inMemory` to create an in-memory SQLite database with PowerSync.
   This may be useful for testing.

README.md

@@ -24,7 +24,14 @@ and API documentation [here](https://powersync-ja.github.io/powersync-kotlin/).
 
 - [core](./core/)
 
-    - This is the Kotlin Multiplatform SDK implementation.
+    - This is the Kotlin Multiplatform SDK implementation, built by depending on `common`
+      and linking SQLite.
+
+- [common](./common/)
+
+    - This is the Kotlin Multiplatform SDK implementation without a dependency on a fixed
+      SQLite bundle. This allows the SDK to be used with custom SQLite installations (like
+      e.g. SQLCipher).
 
 - [integrations](./integrations/)
   - [room](./integrations/room/README.md): Allows using the [Room database library](https://developer.android.com/jetpack/androidx/releases/room) with PowerSync, making it easier to run typed queries on the database.

common/README.md

@@ -0,0 +1,28 @@
+# PowerSync common
+
+This module contains core definitions for the PowerSync SDK, without linking or bundling a SQLite dependency.
+
+This allows the module to be used as a building block for PowerSync SDKs with and without encryption support.
+
+Users should typically depend on `:core` instead.
+
+## Structure
+
+This is a Kotlin Multiplatform project targeting Android, iOS platforms, with the following
+structure:
+
+- `commonMain` - Shared code for all targets, which includes the `PowerSyncBackendConnector`
+  interface and `PowerSyncBuilder` for building a `PowerSync` instance. It also defines
+  the `DatabaseDriverFactory` class to be implemented in each platform.
+- `commonJava` - Shared logic for Android and Java targets.
+- `androidMain` - Android-specific code for loading the core extension.
+- `jvmMain` - Java-specific code for loading the core extension.
+- `nativeMain` - A SQLite driver implemented with cinterop calls to sqlite3.
+- `appleMain`: Utilities for finding a suitable database location on Apple platforms.
+- `appleNonWatchOsMain` and `watchosMain`: Loads the PowerSync core extension (which is linked statically on watchOS
+   and dynamically on other platforms).
+
+## Attachment Helpers
+
+This module contains attachment helpers under the `com.powersync.attachments` package. See
+the [Attachment Helpers README](../common/src/commonMain/kotlin/com/powersync/attachments/README.md)

common/build.gradle.kts

@@ -0,0 +1,277 @@
+import com.powersync.plugins.utils.powersyncTargets
+import de.undercouch.gradle.tasks.download.Download
+import org.gradle.api.tasks.testing.logging.TestExceptionFormat
+import org.gradle.internal.os.OperatingSystem
+import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget
+import org.jetbrains.kotlin.gradle.targets.jvm.tasks.KotlinJvmTest
+import org.jetbrains.kotlin.gradle.tasks.KotlinTest
+import org.jetbrains.kotlin.konan.target.Family
+import java.nio.file.Path
+import kotlin.io.path.createDirectories
+import kotlin.io.path.writeText
+
+plugins {
+    alias(libs.plugins.kotlinMultiplatform)
+    alias(libs.plugins.kotlinSerialization)
+    alias(libs.plugins.android.library)
+    alias(libs.plugins.mavenPublishPlugin)
+    alias(libs.plugins.downloadPlugin)
+    alias(libs.plugins.kotlinter)
+    id("com.powersync.plugins.sonatype")
+    id("com.powersync.plugins.sharedbuild")
+    alias(libs.plugins.mokkery)
+    alias(libs.plugins.kotlin.atomicfu)
+    id("dokka-convention")
+}
+
+val binariesFolder = project.layout.buildDirectory.dir("binaries/desktop")
+val downloadPowersyncDesktopBinaries by tasks.registering(Download::class) {
+    description = "Download PowerSync core extensions for JVM builds and releases"
+
+    val coreVersion =
+        libs.versions.powersync.core
+            .get()
+    val linux_aarch64 =
+        "https://github.com/powersync-ja/powersync-sqlite-core/releases/download/v$coreVersion/libpowersync_aarch64.so"
+    val linux_x64 =
+        "https://github.com/powersync-ja/powersync-sqlite-core/releases/download/v$coreVersion/libpowersync_x64.so"
+    val macos_aarch64 =
+        "https://github.com/powersync-ja/powersync-sqlite-core/releases/download/v$coreVersion/libpowersync_aarch64.dylib"
+    val macos_x64 =
+        "https://github.com/powersync-ja/powersync-sqlite-core/releases/download/v$coreVersion/libpowersync_x64.dylib"
+    val windows_x64 =
+        "https://github.com/powersync-ja/powersync-sqlite-core/releases/download/v$coreVersion/powersync_x64.dll"
+
+    val includeAllPlatformsForJvmBuild =
+        project.findProperty("powersync.binaries.allPlatforms") == "true"
+    val os = OperatingSystem.current()
+
+    // The jar we're releasing for JVM clients needs to include the core extension. For local tests, it's enough to only
+    // download the extension for the OS running the build. For releases, we want to include them all.
+    // We're not compiling native code for JVM builds here (we're doing that for Android only), so we just have to
+    // fetch prebuilt binaries from the powersync-sqlite-core repository.
+    if (includeAllPlatformsForJvmBuild) {
+        src(listOf(linux_aarch64, linux_x64, macos_aarch64, macos_x64, windows_x64))
+    } else {
+        val (aarch64, x64) =
+            when {
+                os.isLinux -> linux_aarch64 to linux_x64
+                os.isMacOsX -> macos_aarch64 to macos_x64
+                os.isWindows -> null to windows_x64
+                else -> error("Unknown operating system: $os")
+            }
+        val arch = System.getProperty("os.arch")
+        src(
+            when (arch) {
+                "aarch64" -> listOfNotNull(aarch64)
+                "amd64", "x86_64" -> listOfNotNull(x64)
+                else -> error("Unsupported architecture: $arch")
+            },
+        )
+    }
+    dest(binariesFolder.map { it.dir("powersync") })
+    onlyIfModified(true)
+}
+
+val generateVersionConstant by tasks.registering {
+    val target = project.layout.buildDirectory.dir("generated/constants")
+    val packageName = "com.powersync.build"
+
+    outputs.dir(target)
+    val currentVersion = version.toString()
+
+    doLast {
+        val dir = target.get().asFile
+        dir.mkdir()
+        val rootPath = dir.toPath()
+
+        val source =
+            """
+            package $packageName
+            
+            internal const val LIBRARY_VERSION: String = "$currentVersion"
+
+            """.trimIndent()
+
+        val packageRoot = packageName.split('.').fold(rootPath, Path::resolve)
+        packageRoot.createDirectories()
+
+        packageRoot.resolve("BuildConstants.kt").writeText(source)
+    }
+}
+
+kotlin {
+    powersyncTargets()
+
+    targets.withType<KotlinNativeTarget> {
+        compilations.named("main") {
+            compileTaskProvider {
+                compilerOptions.freeCompilerArgs.add("-Xexport-kdoc")
+            }
+
+            if (target.konanTarget.family == Family.WATCHOS) {
+                // We're linking the core extension statically, which means that we need a cinterop
+                // to call powersync_init_static
+                cinterops.create("powersync_static") {
+                    packageName("com.powersync.static")
+                    headers(file("src/watchosMain/powersync_static.h"))
+                }
+            }
+
+            cinterops.create("sqlite3") {
+                // We're not linking SQLite here (to allow using this package with e.g. SQLCipher or
+                // SQLite3MultipleCiphers), :core depends on this package and is responsible for bundling SQLite.
+                packageName("com.powersync.internal.sqlite3")
+                includeDirs.allHeaders("src/nativeMain/interop/")
+                definitionFile.set(project.file("src/nativeMain/interop/sqlite3.def"))
+            }
+        }
+    }
+
+    explicitApi()
+
+    applyDefaultHierarchyTemplate()
+    sourceSets {
+        all {
+            languageSettings {
+                optIn("kotlinx.cinterop.ExperimentalForeignApi")
+                optIn("kotlin.time.ExperimentalTime")
+                optIn("kotlin.experimental.ExperimentalObjCRefinement")
+            }
+        }
+
+        val commonIntegrationTest by creating {
+            dependsOn(commonTest.get())
+        }
+
+        val commonJava by creating {
+            dependsOn(commonMain.get())
+        }
+
+        commonMain.configure {
+            kotlin {
+                srcDir(generateVersionConstant)
+            }
+
+            dependencies {
+                api(libs.androidx.sqlite.sqlite)
+
+                implementation(libs.uuid)
+                implementation(libs.kotlin.stdlib)
+                implementation(libs.ktor.client.contentnegotiation)
+                implementation(libs.ktor.serialization.json)
+                implementation(libs.kotlinx.io)
+                implementation(libs.kotlinx.coroutines.core)
+                implementation(libs.kotlinx.datetime)
+                implementation(libs.stately.concurrency)
+                implementation(libs.configuration.annotations)
+                api(libs.ktor.client.core)
+                api(libs.kermit)
+            }
+        }
+
+        androidMain {
+            dependsOn(commonJava)
+            dependencies {
+                api(libs.powersync.sqlite.core.android)
+            }
+        }
+
+        jvmMain {
+            dependsOn(commonJava)
+        }
+
+        // Common apple targets where we link the core extension dynamically
+        val appleNonWatchOsMain by creating {
+            dependsOn(appleMain.get())
+        }
+
+        macosMain.orNull?.dependsOn(appleNonWatchOsMain)
+        iosMain.orNull?.dependsOn(appleNonWatchOsMain)
+        tvosMain.orNull?.dependsOn(appleNonWatchOsMain)
+
+        commonTest.dependencies {
+            implementation(projects.internal.testutils)
+            implementation(libs.kotlin.test)
+        }
+
+        // We're putting the native libraries into our JAR, so integration tests for the JVM can run as part of the unit
+        // tests.
+        jvmTest {
+            dependsOn(commonIntegrationTest)
+        }
+
+        // We have special setup in this build configuration to make these tests link the PowerSync extension, so they
+        // can run integration tests along with the executable for unit testing.
+        appleTest.orNull?.dependsOn(commonIntegrationTest)
+    }
+}
+
+android {
+    compileOptions {
+        targetCompatibility = JavaVersion.VERSION_17
+    }
+
+    buildFeatures {
+        buildConfig = true
+    }
+
+    buildTypes {
+        release {
+            buildConfigField("boolean", "DEBUG", "false")
+        }
+        debug {
+            buildConfigField("boolean", "DEBUG", "true")
+        }
+    }
+
+    namespace = "com.powersync"
+    compileSdk =
+        libs.versions.android.compileSdk
+            .get()
+            .toInt()
+    defaultConfig {
+        minSdk =
+            libs.versions.android.minSdk
+                .get()
+                .toInt()
+        consumerProguardFiles("proguard-rules.pro")
+    }
+
+    ndkVersion = "27.1.12297006"
+}
+
+tasks.named<ProcessResources>(kotlin.jvm().compilations["main"].processResourcesTaskName) {
+    from(downloadPowersyncDesktopBinaries)
+}
+
+// We want to build with recent JDKs, but need to make sure we support Java 8. https://jakewharton.com/build-on-latest-java-test-through-lowest-java/
+val testWithJava8 by tasks.registering(KotlinJvmTest::class) {
+    javaLauncher =
+        javaToolchains.launcherFor {
+            languageVersion = JavaLanguageVersion.of(8)
+        }
+
+    description = "Run tests with Java 8"
+    group = LifecycleBasePlugin.VERIFICATION_GROUP
+
+    // Copy inputs from the normal test task
+    val testTask = tasks.getByName("jvmTest") as KotlinJvmTest
+    classpath = testTask.classpath
+    testClassesDirs = testTask.testClassesDirs
+}
+tasks.named("check").configure { dependsOn(testWithJava8) }
+
+tasks.withType<KotlinTest> {
+    testLogging {
+        events("PASSED", "FAILED", "SKIPPED")
+        exceptionFormat = TestExceptionFormat.FULL
+        showCauses = true
+        showStandardStreams = true
+        showStackTraces = true
+    }
+}
+
+dokka {
+    moduleName.set("PowerSync Common")
+}

core/src/androidMain/kotlin/BuildConfig.kt renamed to common/src/androidMain/kotlin/BuildConfig.kt

@@ -0,0 +1,5 @@
+package com.powersync
+
+@ExperimentalPowerSyncAPI
+@Throws(PowerSyncException::class)
+public actual fun resolvePowerSyncLoadableExtensionPath(): String? = "libpowersync.so"

common/src/androidMain/kotlin/com/powersync/ConnectionFactory.android.kt

@@ -1,16 +1,18 @@
 package com.powersync
 
 import kotlinx.cinterop.UnsafeNumber
-import kotlinx.io.files.FileSystem
 import platform.Foundation.NSApplicationSupportDirectory
 import platform.Foundation.NSBundle
 import platform.Foundation.NSFileManager
 import platform.Foundation.NSSearchPathForDirectoriesInDomains
 import platform.Foundation.NSUserDomainMask
 import kotlin.getValue
 
+/**
+ * The default path to use for databases on Apple platforms.
+ */
 @OptIn(UnsafeNumber::class)
-internal fun appleDefaultDatabasePath(dbFilename: String): String {
+public fun appleDefaultDatabasePath(dbFilename: String): String {
     // This needs to be compatible with https://github.com/touchlab/SQLiter/blob/a37bbe7e9c65e6a5a94c5bfcaccdaae55ad2bac9/sqliter-driver/src/appleMain/kotlin/co/touchlab/sqliter/DatabaseFileContext.kt#L36-L51
     val paths = NSSearchPathForDirectoriesInDomains(NSApplicationSupportDirectory, NSUserDomainMask, true)
     val documentsDirectory = paths[0] as String

core/src/androidMain/kotlin/com/powersync/sync/UserAgent.android.kt renamed to common/src/androidMain/kotlin/com/powersync/sync/UserAgent.android.kt

@@ -0,0 +1,5 @@
+package com.powersync
+
+@ExperimentalPowerSyncAPI
+@Throws(PowerSyncException::class)
+public actual fun resolvePowerSyncLoadableExtensionPath(): String? = powerSyncExtensionPath