-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
4 changed files
with
196 additions
and
158 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| import argparse | ||
| import mimetypes | ||
| import os | ||
| import urllib.request | ||
| from typing import Optional, Sequence | ||
| from urllib.parse import urlparse | ||
|
|
||
| from .common import process_nb, base64_string_to_bytes | ||
|
|
||
|
|
||
| def base64_to_blob_storage( | ||
| base64_string: str, | ||
| az_blob_container_sas_url: str, | ||
| image_path: str) -> (int, str): | ||
| print("Uploading image to blob storage...") | ||
| image_bytes = base64_string_to_bytes(base64_string) | ||
|
|
||
| o = urlparse(az_blob_container_sas_url) | ||
| # Remove first / from path | ||
| if o.path[0] == '/': | ||
| blob_storage_path = o.path[1:] | ||
| else: | ||
| blob_storage_path = o.path | ||
|
|
||
| storage_account = o.scheme + "://" + o.netloc + "/" | ||
| file_name_only = os.path.basename(image_path) | ||
|
|
||
| response_status, url_path = http_put( | ||
| storage_account, blob_storage_path, | ||
| file_name_only, o.query, image_path, image_bytes | ||
| ) | ||
|
|
||
| return response_status, url_path | ||
|
|
||
|
|
||
| def http_put( | ||
| storage_url: str, container_name: str, blob_name: str, | ||
| qry_string: str, image_name: str, image_bytes) -> (int, str): | ||
|
|
||
| file_name_only = os.path.basename(image_name) | ||
|
|
||
| file_ext = os.path.splitext(file_name_only)[1] | ||
|
|
||
| url = storage_url + container_name + '/' + blob_name + '?' + qry_string | ||
|
|
||
| req = urllib.request.Request( | ||
| url, data=image_bytes, method='PUT', | ||
| headers={ | ||
| 'content-type': mimetypes.types_map[file_ext], | ||
| 'x-ms-blob-type': 'BlockBlob' | ||
| }) | ||
| response_code = urllib.request.urlopen(req).code | ||
| # response_code = requests.put( | ||
| # url, | ||
| # data=image_bytes, | ||
| # headers={ | ||
| # 'content-type': mimetypes.types_map[file_ext], | ||
| # 'x-ms-blob-type': 'BlockBlob' | ||
| # }, | ||
| # params={'file': file_name_only} | ||
| # ).status_code | ||
| return response_code, url | ||
|
|
||
|
|
||
| def main(argv: Optional[Sequence[str]] = None) -> int: | ||
| parser = argparse.ArgumentParser() | ||
| parser.add_argument('filenames', nargs='*', help='Filenames to fix') | ||
| parser.add_argument( | ||
| '--az-blob-container-url', | ||
| default=None, | ||
| help='If provided it will upload images to external Azure Blob Storage container rather than local files') # NOQA E501 | ||
| parser.add_argument( | ||
| '--add-changes-to-staging', | ||
| default=False, action='store_true', | ||
| help='Automatically add new and changed files to staging') | ||
| parser.add_argument( | ||
| '--auto-commit-changes', default=False, action='store_true', | ||
| help='Automatically commits added and changed files in staging') | ||
| args = parser.parse_args(argv) | ||
|
|
||
| retv = 0 | ||
|
|
||
| for filename in args.filenames: | ||
| return_value = process_nb(filename=filename, **vars(args)) | ||
| retv |= return_value | ||
|
|
||
| return retv | ||
|
|
||
|
|
||
| if __name__ == '__main__': | ||
| exit(main()) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| import base64 | ||
| import os | ||
| import re | ||
| import subprocess | ||
| import uuid | ||
|
|
||
| from .base64_to_external_storage import base64_to_blob_storage | ||
| from .base64_to_image_files import base64_to_local_file | ||
|
|
||
|
|
||
| def create_nb_cell_output(url: str) -> str: | ||
| return """"text/html": [ | ||
| "<img src=\\"%s\\"/>" | ||
| ]""" % url | ||
|
|
||
|
|
||
| def git_add(filenames: str): | ||
| process = subprocess.Popen( | ||
| ['git', 'add', *filenames.split()], | ||
| shell=True, | ||
| stdout=subprocess.PIPE, | ||
| stderr=subprocess.PIPE) | ||
| stdout, stderr = process.communicate() | ||
|
|
||
|
|
||
| def base64_string_to_bytes(base64_string: str) -> bytes: | ||
| return base64.decodebytes(base64_string.encode()) | ||
|
|
||
|
|
||
| def process_nb( | ||
| filename: str, | ||
| add_changes_to_staging: bool, | ||
| auto_commit_changes: bool, | ||
| az_blob_container_url: str = None, | ||
| **kwargs | ||
| ) -> int: | ||
| print("==================") | ||
| print(add_changes_to_staging, auto_commit_changes) | ||
| print("Processing %s" % filename) | ||
| with open(filename, 'r') as file: | ||
| org_data = " ".join(file.readlines()) | ||
| data = org_data | ||
| matches = re.findall( | ||
| r"\"image/(?:gif|png|jpeg|bmp|webp)\": \".*[a-zA-Z0-9+/=]\"", | ||
| data) | ||
|
|
||
| new_files = "" | ||
|
|
||
| for match in matches: | ||
| ext = "." + re.findall(r"image/[a-zA-Z]*", match)[0].split('/')[1] | ||
| image_path = "nb_images" + "/" + str(uuid.uuid4()) + ext | ||
|
|
||
| full_path = "./" + os.path.dirname(filename) + "/" + image_path | ||
|
|
||
| base64_string = ( | ||
| match.split(':')[1] | ||
| .replace('"', '') | ||
| .replace(' ', '') | ||
| .replace('\\n', '') | ||
| ) | ||
|
|
||
| if az_blob_container_url: | ||
| response_status, url_path = base64_to_blob_storage( | ||
| base64_string, az_blob_container_url, full_path | ||
| ) | ||
|
|
||
| if response_status >= 200 and response_status < 300: | ||
| print(f"Successfully uploaded image to blob storage: {url_path}") # NOQA E501 | ||
| else: | ||
| print(f"Uploading process failed with response code: {response_status}") # NOQA E501 | ||
| else: | ||
| print("Converting base64 to image file and saving as %s" % full_path) # NOQA E501 | ||
| base64_to_local_file( | ||
| base64_string, full_path | ||
| ) | ||
| url_path = "./" + image_path | ||
| new_files += " " + full_path | ||
|
|
||
| data = data.replace(match, create_nb_cell_output(url_path)) | ||
|
|
||
| if org_data != data: | ||
| with open(filename, 'w') as file: | ||
| file.write(data) | ||
| new_files += " " + filename | ||
| new_files = new_files.strip() | ||
|
|
||
| if add_changes_to_staging: | ||
| print("'--add_changes_to_staging' flag set to 'True' - adding new and changed files to staging...") # NOQA E501 | ||
| print(new_files) | ||
| git_add(new_files) | ||
|
|
||
| if auto_commit_changes: | ||
| print("'--auto_commit_changes' flag set to 'True' - git hook set to return exit code 0.") # NOQA E501 | ||
| return 0 | ||
|
|
||
| return 1 | ||
| else: | ||
| print("Didn't find any base64 strings...") | ||
| return 0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters