Skip to content

Commit

Permalink
Merge pull request #268 from ipfs/branch-v0.23.0
Browse files Browse the repository at this point in the history
Update to Kubo v0.23.0 - add prebuilt-binary pipeline
  • Loading branch information
hsanjuan authored Oct 6, 2023
2 parents 07312ee + 8ca41f8 commit 6b09374
Show file tree
Hide file tree
Showing 11 changed files with 616 additions and 1,550 deletions.
33 changes: 0 additions & 33 deletions .github/workflows/plugin-test.yml

This file was deleted.

120 changes: 120 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
name: Kubo plugin

on:
push:


env:
# must be same as official ipfs builds. See distributions/.tool-versions
GO_VERSION: "1.21.1"

jobs:
build-artifacts:
name: "Build"
strategy:
matrix:
os:
- ubuntu-20.04
- macos-latest
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 2
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: ${{ env.GO_VERSION }}
- name: Extract release name from tag
run: |
RELEASE=$(basename ${{ github.ref_name }})
echo "RELEASE=$RELEASE" >> "$GITHUB_ENV"
- name: Package plugin
run: make dist-plugin
- name: Record Go environment
run: |
echo "GOHOSTOS=$(go env GOHOSTOS)" >> "$GITHUB_ENV"
echo "GOHOSTARCH=$(go env GOHOSTARCH)" >> "$GITHUB_ENV"
- name: Rename package
run: |
NAME="go-ds-s3-plugin_${{ env.RELEASE }}_${{ env.GOHOSTOS }}_${{ env.GOHOSTARCH }}.tar.gz"
mv go-ds-s3-plugin/go-ds-s3-plugin.tar.gz "$NAME"
echo "ARTIFACT_NAME=$NAME" >> "$GITHUB_ENV"
- name: Archive artifacts
uses: actions/upload-artifact@v3
with:
name: ${{ env.ARTIFACT_NAME }}
path: go-ds-s3-plugin_*.tar.gz

test-artifacts:
name: "Test"
needs: build-artifacts
strategy:
matrix:
os:
- ubuntu-20.04
# macos test fail with dist-built ipfs.
# - macos-latest
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 2
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: ${{ env.GO_VERSION }}
- name: Record variables
run: |
V=`cat go.mod | grep github.com/ipfs/kubo | grep -o 'v.*'`
echo "KUBO_VERSION=$V" >> "$GITHUB_ENV"
GOHOSTOS=`go env GOHOSTOS`
echo "GOHOSTOS=$GOHOSTOS" >> "$GITHUB_ENV"
RELEASE=$(basename ${{ github.ref_name }})
echo "RELEASE=$RELEASE" >> "$GITHUB_ENV"
echo PLUGIN_ARTIFACT=go-ds-s3-plugin_${RELEASE}_${GOHOSTOS}_amd64.tar.gz >> "$GITHUB_ENV"
- name: Download artifact
id: download
uses: actions/download-artifact@v3
with:
name: ${{ env.PLUGIN_ARTIFACT }}
- name: Download unpack Kubo
run: |
wget -nv https://dist.ipfs.tech/kubo/${{ env.KUBO_VERSION }}/kubo_${{ env.KUBO_VERSION }}_${{ env.GOHOSTOS }}-amd64.tar.gz
tar -xf kubo_${{ env.KUBO_VERSION }}_${{ env.GOHOSTOS }}-amd64.tar.gz
tar -xf go-ds-s3-plugin_*.tar.gz
chmod +x kubo/ipfs
- name: Initialize IPFS and copy plugin
run: |
export IPFS_PATH=$(pwd)/ipfs-config
echo "IPFS_PATH=$IPFS_PATH" >> "$GITHUB_ENV"
./kubo/ipfs init
mkdir -p ipfs-config/plugins
cp go-ds-s3-plugin/go-ds-s3-plugin ipfs-config/plugins/
- name: Check IPFS works with the plugin
run: ./kubo/ipfs version --all

release:
name: "Release"
needs: test-artifacts
runs-on: ubuntu-20.04
steps:
- name: Download artifacts
id: download
uses: actions/download-artifact@v3
- name: Extract release name from tag
run: |
RELEASE=$(basename ${{ github.ref_name }})
echo "RELEASE=$RELEASE" >> "$GITHUB_ENV"
- name: Release
uses: softprops/action-gh-release@v1
if: startsWith(github.ref, 'refs/tags/go-ds-s3-plugin/v')
with:
files: go-ds-s3-plugin_*.tar.gz/*
body: |
This is a binary build of the go-ds-s3 Kubo plugin targeting Kubo version ${{ env.RELEASE }}.
To install, download the relevant asset for your platform, unpack the plugin file (`go-ds-s3-plugin`) and place it in `~/.ipfs/plugins`. MacOS users will need to compile Kubo themselves, as the official releases have no CGO support.
See the included README.md for more information.
name: ${{ github.ref_name }}
45 changes: 11 additions & 34 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,38 +1,15 @@
# Force Go Modules
GO111MODULE = on
plugin:
$(MAKE) -C go-ds-s3-plugin all

GOCC ?= go
GOFLAGS ?=
install-plugin:
$(MAKE) -C go-ds-s3-plugin install

# If set, override the install location for plugins
IPFS_PATH ?= $(HOME)/.ipfs
dist-plugin:
$(MAKE) -C go-ds-s3-plugin dist

# If set, override the IPFS version to build against. This _modifies_ the local
# go.mod/go.sum files and permanently sets this version.
IPFS_VERSION ?= $(lastword $(shell $(GOCC) list -m github.com/ipfs/kubo))
check:
go vet ./...
staticcheck --checks all ./...
misspell -error -locale US .

# make reproducible
ifneq ($(findstring /,$(IPFS_VERSION)),)
# Locally built kubo
GOFLAGS += -asmflags=all=-trimpath="$(GOPATH)" -gcflags=all=-trimpath="$(GOPATH)"
else
# Remote version of kubo (e.g. via `go get -trimpath` or official distribution)
GOFLAGS += -trimpath
endif

.PHONY: install build

go.mod: FORCE
./set-target.sh $(IPFS_VERSION)

FORCE:

s3plugin.so: plugin/main/main.go go.mod
CGO_ENABLED=1 $(GOCC) build $(GOFLAGS) -buildmode=plugin -o "$@" "$<"
chmod +x "$@"

build: s3plugin.so
@echo "Built against" $(IPFS_VERSION)

install: build
install -Dm700 s3plugin.so "$(IPFS_PATH)/plugins/go-ds-s3.so"
.PHONY: plugin install-plugin check
102 changes: 28 additions & 74 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,38 @@ This is an implementation of the datastore interface backed by amazon s3.

**NOTE:** Plugins only work on Linux and MacOS at the moment. You can track the progress of this issue here: https://github.com/golang/go/issues/19282

## Building and Installing
## Quickstart

You must build the plugin with the *exact* version of go used to build the kubo binary you will use it with. You can find the go version for kubo builds from dist.ipfs.io in the build-info file, e.g. https://dist.ipfs.io/kubo/v0.4.22/build-info or by running `ipfs version --all`.
1. Grab a plugin release from the [releases](https://github.com/ipfs/go-ds-s3/releases) section matching your Kubo version and install the plugin file in `~/.ipfs/plugins`.
2. Follow the instructions in the plugin's [README.md](go-ds-s3-plugin/README.md)

In addition to needing the exact version of go, you need to build the correct version of this plugin.

* To build against a released version of kubo, checkout the `release/v$VERSION` branch and build.
* To build against a custom (local) build of kubo, run `make IPFS_VERSION=/path/to/kubo/source`.
## Building and installing

You can then install it into your local IPFS repo by running `make install`.

The plugin can be manually built/installed for different versions of Kubo (starting with 0.23.0) with:

```
git checkout go-ds-s3-plugin/v<kubo-version>
make plugin
make install-plugin
```

## Updating to a new version

1. `go get` the Kubo release you want to build for. Make sure any other
dependencies are aligned to what Kubo uses.
2. `make install` and test.


If you are building against dist-released versions of Kubo, you need to build using the same version of go that was used to build the release ([here](https://github.com/ipfs/distributions/blob/master/.tool-versions)).

If you are building against your own build of Kubo you must align your plugin to use it.

If you are updating this repo to produce a new version of the plugin:

1. Submit a PR so that integration tests run
2. Make a new tag `go-ds-s3-plugin/v<kubo_version>` and push it. This will build and release the plugin prebuilt binaries.

## Bundling

Expand Down Expand Up @@ -46,74 +68,6 @@ As go plugins can be finicky to correctly compile and install, you may want to c
> make install
```

## Detailed Installation

For a brand new ipfs instance (no data stored yet):

1. Copy s3plugin.so $IPFS_DIR/plugins/go-ds-s3.so (or run `make install` if you are installing locally).
2. Run `ipfs init`.
3. Edit $IPFS_DIR/config to include s3 details (see Configuration below).
4. Overwrite `$IPFS_DIR/datastore_spec` as specified below (*Don't do this on an instance with existing data - it will be lost*).

### Configuration

The config file should include the following:
```json
{
"Datastore": {
...

"Spec": {
"mounts": [
{
"child": {
"type": "s3ds",
"region": "us-east-1",
"bucket": "$bucketname",
"rootDirectory": "$bucketsubdirectory",
"accessKey": "",
"secretKey": ""
},
"mountpoint": "/blocks",
"prefix": "s3.datastore",
"type": "measure"
},
```

If the access and secret key are blank they will be loaded from the usual ~/.aws/.
If you are on another S3 compatible provider, e.g. Linode, then your config should be:

```json
{
"Datastore": {
...

"Spec": {
"mounts": [
{
"child": {
"type": "s3ds",
"region": "us-east-1",
"bucket": "$bucketname",
"rootDirectory": "$bucketsubdirectory",
"regionEndpoint": "us-east-1.linodeobjects.com",
"accessKey": "",
"secretKey": ""
},
"mountpoint": "/blocks",
"prefix": "s3.datastore",
"type": "measure"
},
```

If you are configuring a brand new ipfs instance without any data, you can overwrite the datastore_spec file with:

```
{"mounts":[{"bucket":"$bucketname","mountpoint":"/blocks","region":"us-east-1","rootDirectory":"$bucketsubdirectory"},{"mountpoint":"/","path":"datastore","type":"levelds"}],"type":"mount"}
```

Otherwise, you need to do a datastore migration.

## Contribute

Feel free to join in. All welcome. Open an [issue](https://github.com/ipfs/go-ipfs-example-plugin/issues)!
Expand Down
23 changes: 23 additions & 0 deletions go-ds-s3-plugin/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
SRC := $(shell find ../.. -type f -name '*.go')
IPFS_PATH ?= $(HOME)/.ipfs

export CGO_ENABLED := 1

all: build

build: $(SRC)
go build -buildmode=plugin -trimpath -o go-ds-s3-plugin

install: build
mkdir -p "$(IPFS_PATH)/.ipfs/plugins"
install -Dm700 go-ds-s3-plugin "$(IPFS_PATH)/plugins/go-ds-s3-plugin"

dist: build
mkdir -p dist/go-ds-s3-plugin
cp go-ds-s3-plugin README.md dist/go-ds-s3-plugin/
chmod +x dist/go-ds-s3-plugin/go-ds-s3-plugin
tar -C dist -zcf go-ds-s3-plugin.tar.gz go-ds-s3-plugin
rm -rf go-ds-s3-plugin
echo "Packaged as go-ds-s3-plugin.tar.gz"

.PHONY: install
Loading

0 comments on commit 6b09374

Please sign in to comment.