Skip to content

Commit

Permalink
Only load gcs client when needed since this requires GCP credential (#…
Browse files Browse the repository at this point in the history
…202)

* Only load gcs client when needed since this requires GCP credential

* remove logging
  • Loading branch information
shifucun authored Oct 10, 2023
1 parent d383700 commit 2aa387a
Show file tree
Hide file tree
Showing 2 changed files with 5 additions and 6 deletions.
1 change: 0 additions & 1 deletion simple/stats/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
_CODEDIR = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(1, os.path.join(_CODEDIR, "../"))

from util import dc_client as dc

FLAGS = flags.FLAGS

Expand Down
10 changes: 5 additions & 5 deletions simple/util/filehandler.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""A generic FileHandler abstraction that allows clients to work seamlessly with
"""A generic FileHandler abstraction that allows clients to work seamlessly with
local and GCS files and directories.
"""

import os
import logging
import io
import os
from google.cloud import storage

_GCS_PATH_PREFIX = "gs://"
Expand Down Expand Up @@ -94,15 +95,14 @@ def list_files(self, extension: str = None) -> list[str]:


class GcsFileHandler(FileHandler):
gcs_client = storage.Client()
# Using print instead of logging since the class is loaded before logging is initialized.
print("Using GCP Project:", gcs_client.project)

def __init__(self, path: str) -> None:
if not path.startswith(_GCS_PATH_PREFIX):
raise ValueError(f"Expected {_GCS_PATH_PREFIX} prefix, got {path}")
bucket_name, blob_name = path[len(_GCS_PATH_PREFIX):].split('/', 1)
self.bucket = GcsFileHandler.gcs_client.bucket(bucket_name)
gcs_client = storage.Client()
self.bucket = gcs_client.bucket(bucket_name)
self.blob = self.bucket.blob(blob_name)
isdir = path.endswith("/")
super().__init__(path, isdir)
Expand Down

0 comments on commit 2aa387a

Please sign in to comment.