Skip to content

Commit

Permalink
Fix image uploading to S3 (#167)
Browse files Browse the repository at this point in the history
* Fix image uploading to S3

* image refactoring stuff

* fix test

* fix more mock
codekansas authored Jul 24, 2024
1 parent dbaf6fd commit b89547d
Showing 16 changed files with 164 additions and 139 deletions.
26 changes: 19 additions & 7 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
@@ -91,31 +91,43 @@ Settings for the app backend live in the `store/settings/` directory. To configu
To locally develop, setting the following environment variables will work (presuming you have set everything else up):

```
# Specifies a local environment verses production environment.
export ROBOLIST_ENVIRONMENT=local
# For AWS
export AWS_DEFAULT_REGION='us-east-1'
export AWS_ACCESS_KEY_ID=test
export AWS_SECRET_ACCESS_KEY=test
export AWS_ENDPOINT_URL_DYNAMODB=http://127.0.0.1:4566
export AWS_ENDPOINT_URL_S3=http://127.0.0.1:4566
# For letting the frontend know the backend URL.
export REACT_APP_BACKEND_URL=http://127.0.0.1:8080
export ROBOLIST_SMTP_HOST=smtp.gmail.com
export ROBOLIST_SMTP_SENDER_EMAIL=
export ROBOLIST_SMTP_PASSWORD=
export ROBOLIST_SMTP_SENDER_NAME=
export ROBOLIST_SMTP_USERNAME=
# For SMTP
export SMTP_HOST=smtp.gmail.com
export SMTP_SENDER_EMAIL=
export SMTP_PASSWORD=
export SMTP_SENDER_NAME=
export SMTP_USERNAME=
# For Github OAuth
export GITHUB_CLIENT_ID=
export GITHUB_CLIENT_SECRET=
# For Google OAuth
export GOOGLE_CLIENT_ID=
export GOOGLE_CLIENT_SECRET=
```

### Github OAuth Configuration

To run Github OAuth locally, you must follow these steps:

1. Create an OAuth App on [Github Developer Settings](https://github.com/settings/developers)
2. Set both Homepage URL and Authorization callback URL to `http://127.0.0.1:3000` before you `Update application` on Github Oauth App configuration
3. Copy the Client ID and Client Secret from Github OAuth App configuration and set them in `GITHUB_CLIENT_ID` and `GITHUB_CLIENT_SECRET` respectively
4. Run `source env.sh` in your Fast API terminal window to ensure it has access to the github environment variables


## React

To install the React dependencies, use [nvm](https://github.com/nvm-sh/nvm) and [npm](https://www.npmjs.com/):
4 changes: 2 additions & 2 deletions frontend/src/components/files/ViewImage.tsx
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { S3_URL } from "constants/backend";
import { BACKEND_URL } from "constants/backend";
import React from "react";

interface ImageProps {
@@ -10,7 +10,7 @@ const ImageComponent: React.FC<ImageProps> = ({ imageId, caption }) => {
return (
<div style={{ width: "100%", paddingTop: "100%", position: "relative" }}>
<img
src={new URL("images/" + imageId, S3_URL).toString()}
src={new URL(`image/${imageId}`, BACKEND_URL).toString()}
alt={caption}
className="d-block rounded-lg"
style={{
2 changes: 0 additions & 2 deletions frontend/src/constants/backend.ts
Original file line number Diff line number Diff line change
@@ -3,8 +3,6 @@ import { AxiosError, isAxiosError } from "axios";
export const BACKEND_URL =
process.env.REACT_APP_BACKEND_URL || "http://127.0.0.1:8080";

export const S3_URL = process.env.S3_URL || "http://127.0.0.1:4566";

// eslint-disable-next-line
export const humanReadableError = (error: any | undefined) => {
if (isAxiosError(error)) {
4 changes: 2 additions & 2 deletions frontend/src/hooks/api.tsx
Original file line number Diff line number Diff line change
@@ -53,7 +53,7 @@ export class api {

public async send_register_github(): Promise<string> {
try {
const res = await this.api.get("/users/github-login");
const res = await this.api.get("/users/github/login");
return res.data;
} catch (error) {
if (axios.isAxiosError(error)) {
@@ -70,7 +70,7 @@ export class api {

public async login_github(code: string): Promise<MeResponse> {
try {
const res = await this.api.get(`/users/github-code/${code}`);
const res = await this.api.get(`/users/github/code/${code}`);
return res.data;
} catch (error) {
if (axios.isAxiosError(error)) {
4 changes: 3 additions & 1 deletion store/app/crud/robots.py
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@

from store.app.crud.base import BaseCrud
from store.app.model import Part, Robot
from store.settings import settings

logger = logging.getLogger(__name__)

@@ -45,4 +46,5 @@ async def list_your_parts(self, user_id: str, page: int, search_query: str) -> t
return await self._list_your(Part, user_id, page, lambda x: x.timestamp, search_query)

async def upload_image(self, file: UploadFile) -> None:
await (await self.s3.Bucket("images")).upload_fileobj(file.file, file.filename or "")
bucket = await self.s3.Bucket(settings.s3.bucket)
await bucket.upload_fileobj(file.file, f"{settings.s3.prefix}{file.filename}")
27 changes: 0 additions & 27 deletions store/app/model.py
Original file line number Diff line number Diff line change
@@ -87,33 +87,6 @@ def from_jwt(cls, jwt_token: str) -> Self:
return cls(id=data["token"], user_id=data["user_id"])


class RegisterToken(RobolistBaseModel):
"""Stores a token for registering a new user."""

email: str

@classmethod
def create(cls, email: str) -> Self:
return cls(
id=str(new_uuid()),
email=email,
)

def to_jwt(self) -> str:
return jwt.encode(
payload={"token": self.id, "email": self.email},
key=settings.crypto.jwt_secret,
)

@classmethod
def from_jwt(cls, jwt_token: str) -> Self:
data = jwt.decode(
jwt=jwt_token,
key=settings.crypto.jwt_secret,
)
return cls(id=data["token"], email=data["email"])


class Bom(BaseModel):
part_id: str
quantity: int
Empty file.
100 changes: 100 additions & 0 deletions store/app/routers/auth/github.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
"""Defines the API endpoint for authenticating with Github."""

import logging
from typing import Annotated

from fastapi import APIRouter, Depends, Response
from httpx import AsyncClient, Response as HttpxResponse
from pydantic.main import BaseModel

from store.app.db import Crud
from store.app.model import UserPermissions
from store.settings import settings

logger = logging.getLogger(__name__)

github_auth_router = APIRouter()


@github_auth_router.get("/login")
async def github_login() -> str:
"""Gives the user a redirect url to login with github.
Returns:
Github oauth redirect url.
"""
return f"https://github.com/login/oauth/authorize?scope=user:email&client_id={settings.oauth.github_client_id}"


async def github_access_token_req(params: dict[str, str], headers: dict[str, str]) -> HttpxResponse:
async with AsyncClient() as client:
return await client.post(
url="https://github.com/login/oauth/access_token",
params=params,
headers=headers,
)


async def github_req(headers: dict[str, str]) -> HttpxResponse:
async with AsyncClient() as client:
return await client.get("https://api.github.com/user", headers=headers)


async def github_email_req(headers: dict[str, str]) -> HttpxResponse:
async with AsyncClient() as client:
return await client.get("https://api.github.com/user/emails", headers=headers)


class UserInfoResponse(BaseModel):
id: str
permissions: UserPermissions


@github_auth_router.get("/code/{code}", response_model=UserInfoResponse)
async def github_code(
code: str,
crud: Annotated[Crud, Depends(Crud.get)],
response: Response,
) -> UserInfoResponse:
"""Gives the user a session token upon successful github authentication and creation of user.
Args:
code: Github code returned from the successful authentication.
crud: The CRUD object.
response: The response object.
Returns:
UserInfoResponse.
"""
params = {
"client_id": settings.oauth.github_client_id,
"client_secret": settings.oauth.github_client_secret,
"code": code,
}
headers = {"Accept": "application/json"}
oauth_response = await github_access_token_req(params, headers)
response_json = oauth_response.json()

# access token is used to retrieve user oauth details
access_token = response_json["access_token"]
headers.update({"Authorization": f"Bearer {access_token}"})
oauth_response = await github_req(headers)
oauth_email_response = await github_email_req(headers)

github_id = oauth_response.json()["html_url"]
email = next(entry["email"] for entry in oauth_email_response.json() if entry["primary"])

user = await crud.get_user_from_github_token(github_id)

# We create a new user if the user does not exist yet.
if user is None:
user = await crud.create_user_from_github_token(
email=email,
github_id=github_id,
)

api_key = await crud.add_api_key(user.id)

response.set_cookie(key="session_token", value=api_key.id, httponly=True, samesite="lax")

return UserInfoResponse(id=user.id, permissions=user.permissions)
9 changes: 8 additions & 1 deletion store/app/routers/image.py
Original file line number Diff line number Diff line change
@@ -5,11 +5,12 @@
from typing import Annotated

from fastapi import APIRouter, Depends, HTTPException, UploadFile
from fastapi.responses import JSONResponse
from fastapi.responses import JSONResponse, RedirectResponse
from PIL import Image

from store.app.crypto import new_uuid
from store.app.db import Crud
from store.settings import settings

image_router = APIRouter()

@@ -38,3 +39,9 @@ async def upload_image(crud: Annotated[Crud, Depends(Crud.get)], file: UploadFil
return JSONResponse(status_code=200, content={"id": image_id})
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))


@image_router.get("{image_fname}")
async def image_url(image_fname: str) -> RedirectResponse:
image_url = f"{settings.site.image_base_url}/{image_fname}"
return RedirectResponse(url=image_url)
87 changes: 4 additions & 83 deletions store/app/routers/users.py
Original file line number Diff line number Diff line change
@@ -6,13 +6,12 @@

from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response, status
from fastapi.security.utils import get_authorization_scheme_param
from httpx import AsyncClient, Response as HttpxResponse
from pydantic.main import BaseModel as PydanticBaseModel

from store.app.db import Crud
from store.app.model import UserPermissions
from store.app.routers.auth.github import github_auth_router
from store.app.utils.email import send_delete_email
from store.settings import settings

logger = logging.getLogger(__name__)

@@ -132,90 +131,12 @@ async def get_users_batch_endpoint(
return [PublicUserInfoResponse(id=user.id, email=user.email) for user in users]


@users_router.get("/github-login")
async def github_login() -> str:
"""Gives the user a redirect url to login with github.
Returns:
Github oauth redirect url.
"""
return f"https://github.com/login/oauth/authorize?scope=user:email&client_id={settings.oauth.github_client_id}"


async def github_access_token_req(params: dict[str, str], headers: dict[str, str]) -> HttpxResponse:
async with AsyncClient() as client:
return await client.post(
url="https://github.com/login/oauth/access_token",
params=params,
headers=headers,
)


async def github_req(headers: dict[str, str]) -> HttpxResponse:
async with AsyncClient() as client:
return await client.get("https://api.github.com/user", headers=headers)


async def github_email_req(headers: dict[str, str]) -> HttpxResponse:
async with AsyncClient() as client:
return await client.get("https://api.github.com/user/emails", headers=headers)


@users_router.get("/github-code/{code}", response_model=UserInfoResponse)
async def github_code(
code: str,
crud: Annotated[Crud, Depends(Crud.get)],
response: Response,
) -> UserInfoResponse:
"""Gives the user a session token upon successful github authentication and creation of user.
Args:
code: Github code returned from the successful authentication.
crud: The CRUD object.
response: The response object.
Returns:
UserInfoResponse.
"""
params = {
"client_id": settings.oauth.github_client_id,
"client_secret": settings.oauth.github_client_secret,
"code": code,
}
headers = {"Accept": "application/json"}
oauth_response = await github_access_token_req(params, headers)
response_json = oauth_response.json()

# access token is used to retrieve user oauth details
access_token = response_json["access_token"]
headers.update({"Authorization": f"Bearer {access_token}"})
oauth_response = await github_req(headers)
oauth_email_response = await github_email_req(headers)

github_id = oauth_response.json()["html_url"]
email = next(entry["email"] for entry in oauth_email_response.json() if entry["primary"])

user = await crud.get_user_from_github_token(github_id)
# Exception occurs when user does not exist.
# Create a user if this is the case.
if user is None:
user = await crud.create_user_from_github_token(
email=email,
github_id=github_id,
)
# This is solely so mypy stops complaining.
assert user is not None

api_key = await crud.add_api_key(user.id)

response.set_cookie(key="session_token", value=api_key.id, httponly=True, samesite="lax")

return UserInfoResponse(id=user.id, permissions=user.permissions)


@users_router.get("/{id}", response_model=PublicUserInfoResponse)
async def get_user_info_by_id_endpoint(id: str, crud: Annotated[Crud, Depends(Crud.get)]) -> PublicUserInfoResponse:
user = await crud.get_user(id)
if user is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="User not found")
return PublicUserInfoResponse(id=user.id, email=user.email)


users_router.include_router(github_auth_router, prefix="/github", tags=["github"])
5 changes: 4 additions & 1 deletion store/settings/configs/local.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
crypto:
jwt_secret: fakeJwtSecret
s3:
bucket: images
prefix: ""
site:
homepage: http://127.0.0.1:3000
image_url: http://127.0.0.1:4566/images
image_base_url: http://127.0.0.1:4566/images
2 changes: 1 addition & 1 deletion store/settings/configs/production.yaml
Original file line number Diff line number Diff line change
@@ -2,4 +2,4 @@ crypto:
jwt_secret: ${oc.env:JWT_SECRET}
site:
homepage: https://robolist.xyz
image_url: https://media.robolist.xyz
image_base_url: https://media.robolist.xyz
21 changes: 15 additions & 6 deletions store/settings/environment.py
Original file line number Diff line number Diff line change
@@ -9,6 +9,8 @@
class OauthSettings:
github_client_id: str = field(default=II("oc.env:GITHUB_CLIENT_ID"))
github_client_secret: str = field(default=II("oc.env:GITHUB_CLIENT_SECRET"))
google_client_id: str = field(default=II("oc.env:GOOGLE_CLIENT_ID"))
google_client_secret: str = field(default=II("oc.env:GOOGLE_CLIENT_SECRET"))


@dataclass
@@ -27,18 +29,24 @@ class UserSettings:

@dataclass
class EmailSettings:
host: str = field(default=II("oc.env:ROBOLIST_SMTP_HOST"))
host: str = field(default=II("oc.env:SMTP_HOST"))
port: int = field(default=587)
username: str = field(default=II("oc.env:ROBOLIST_SMTP_USERNAME"))
password: str = field(default=II("oc.env:ROBOLIST_SMTP_PASSWORD"))
sender_email: str = field(default=II("oc.env:ROBOLIST_SMTP_SENDER_EMAIL"))
sender_name: str = field(default=II("oc.env:ROBOLIST_SMTP_SENDER_NAME"))
username: str = field(default=II("oc.env:SMTP_USERNAME"))
password: str = field(default=II("oc.env:SMTP_PASSWORD"))
sender_email: str = field(default=II("oc.env:SMTP_SENDER_EMAIL"))
sender_name: str = field(default=II("oc.env:SMTP_SENDER_NAME"))


@dataclass
class S3Settings:
bucket: str = field(default=II("oc.env:S3_BUCKET"))
prefix: str = field(default=II("oc.env:S3_PREFIX"))


@dataclass
class SiteSettings:
homepage: str = field(default=MISSING)
image_url: str | None = field(default=None)
image_base_url: str = field(default=MISSING)


@dataclass
@@ -47,5 +55,6 @@ class EnvironmentSettings:
user: UserSettings = field(default_factory=UserSettings)
crypto: CryptoSettings = field(default_factory=CryptoSettings)
email: EmailSettings = field(default_factory=EmailSettings)
s3: S3Settings = field(default_factory=S3Settings)
site: SiteSettings = field(default_factory=SiteSettings)
debug: bool = field(default=False)
6 changes: 3 additions & 3 deletions tests/conftest.py
Original file line number Diff line number Diff line change
@@ -80,20 +80,20 @@ def mock_send_email(mocker: MockerFixture) -> MockType:

@pytest.fixture(autouse=True)
def mock_github_access_token(mocker: MockerFixture) -> MockType:
mock = mocker.patch("store.app.routers.users.github_access_token_req")
mock = mocker.patch("store.app.routers.auth.github.github_access_token_req")
mock.return_value = Response(status_code=200, json={"access_token": ""})
return mock


@pytest.fixture(autouse=True)
def mock_github(mocker: MockerFixture) -> MockType:
mock = mocker.patch("store.app.routers.users.github_req")
mock = mocker.patch("store.app.routers.auth.github.github_req")
mock.return_value = Response(status_code=200, json={"html_url": "https://github.com/chennisden"})
return mock


@pytest.fixture(autouse=True)
def mock_github_email(mocker: MockerFixture) -> MockType:
mock = mocker.patch("store.app.routers.users.github_email_req")
mock = mocker.patch("store.app.routers.auth.github.github_email_req")
mock.return_value = Response(status_code=200, json=[{"email": "dchen@kscale.dev", "primary": True}])
return mock
2 changes: 1 addition & 1 deletion tests/test_robots.py
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@ async def test_robots(app_client: AsyncClient) -> None:
await create_tables()

# Register.
response = await app_client.get("/users/github-code/doesnt-matter")
response = await app_client.get("/users/github/code/doesnt-matter")
assert response.status_code == 200, response.json()
assert "session_token" in response.cookies

4 changes: 2 additions & 2 deletions tests/test_users.py
Original file line number Diff line number Diff line change
@@ -18,7 +18,7 @@ async def test_user_auth_functions(app_client: AsyncClient) -> None:
assert response.status_code == 401, response.json()

# Because of the way we patched GitHub functions for mocking, it doesn't matter what token we pass in.
response = await app_client.get("/users/github-code/doesnt-matter")
response = await app_client.get("/users/github/code/doesnt-matter")
assert response.status_code == 200, response.json()
assert "session_token" in response.cookies
token = response.cookies["session_token"]
@@ -49,7 +49,7 @@ async def test_user_auth_functions(app_client: AsyncClient) -> None:
assert response.json()["detail"] == "Not authenticated"

# Log the user back in, getting new session token.
response = await app_client.get("/users/github-code/doesnt-matter")
response = await app_client.get("/users/github/code/doesnt-matter")
assert response.status_code == 200, response.json()
assert "session_token" in response.cookies

0 comments on commit b89547d

Please sign in to comment.