Source code for tiled.client.context

import contextlib
import getpass
import os
import re
import sys
import time
import urllib.parse
import warnings
from pathlib import Path

import httpx
import msgpack

from .._version import get_versions
from ..utils import DictView
from .auth import (
    DEFAULT_TOKEN_CACHE,
    CannotRefreshAuthentication,
    TiledAuth,
    build_refresh_request,
)
from .cache import Revalidate
from .utils import (
    DEFAULT_ACCEPTED_ENCODINGS,
    DEFAULT_TIMEOUT_PARAMS,
    EVENT_HOOKS,
    NotAvailableOffline,
    handle_error,
)

tiled_version = get_versions()["version"]
USER_AGENT = f"python-tiled/{tiled_version}"
API_KEY_AUTH_HEADER_PATTERN = re.compile(r"^Apikey (\w+)$")


[docs]class Context: """ Wrap an httpx.Client with an optional cache and authentication functionality. """
[docs] def __init__( self, uri, *, headers=None, api_key=None, cache=None, offline=False, timeout=None, verify=True, token_cache=DEFAULT_TOKEN_CACHE, app=None, ): # The uri is expected to reach the root API route. uri = httpx.URL(uri) headers = headers or {} headers.setdefault("accept-encoding", ",".join(DEFAULT_ACCEPTED_ENCODINGS)) # Set the User Agent to help the server fail informatively if the client # version is too old. headers.setdefault("user-agent", USER_AGENT) # If ?api_key=... is present, move it from the query into a header. # The server would accept it in the query parameter, but using # a header is a little more secure (e.g. not logged). parsed_params = urllib.parse.parse_qs(uri.query.decode()) api_key_list = parsed_params.pop("api_key", None) if api_key_list is not None: if api_key is not None: raise ValueError( "api_key was provided as query parameter in URI and as keyword argument. Pick one." ) if len(api_key_list) != 1: raise ValueError("Cannot handle two api_key query parameters") (api_key,) = api_key_list if api_key is None: # Check for an API key from the environment. api_key = os.getenv("TILED_API_KEY") # We will set the API key via the `api_key` property below, # after constructing the Client object. # FastAPI redirects /api -> /api/ so add it here to save a request. path = uri.path if not path.endswith("/"): path = f"{path}/" # Construct the uri *without* api_key param. # Drop any params/fragments. self.api_uri = httpx.URL( urllib.parse.urlunsplit((uri.scheme, uri.netloc.decode(), path, {}, "")) ) if timeout is None: timeout = httpx.Timeout(**DEFAULT_TIMEOUT_PARAMS) if app is None: client = httpx.Client( verify=verify, event_hooks=EVENT_HOOKS, timeout=timeout, headers=headers, follow_redirects=True, ) else: from ._testclient import TestClient # verify parameter is dropped, as there is no SSL in ASGI mode client = TestClient( app=app, event_hooks=EVENT_HOOKS, timeout=timeout, headers=headers, ) client.follow_redirects = True client.__enter__() # The TestClient is meant to be used only as a context manager, # where the context starts and stops and the wrapped ASGI app. # We are abusing it slightly to enable interactive use of the # TestClient. # # Given a blank slate, we may not have chosen to support this at # all; we may insist on using this only within a context manager. # But for now it is necessary to suppot a smooth transition for # databroker. We may revisit it later. if sys.version_info < (3, 9): import atexit atexit.register(client.__exit__) else: # The threading module has its own (internal) atexit # mechanism that runs at thread shutdown, prior to the atexit # mechanism that runs at interpreter shutdown. # We need to intervene at that layer to close the portal, or else # we will wait forever for a thread run by the portal to join(). import threading threading._register_atexit(client.__exit__) self.http_client = client self._verify = verify self._cache = cache self._revalidate = Revalidate.IF_WE_MUST self._offline = offline self._token_cache = Path(token_cache) # Make an initial "safe" request to: # (1) Get the server_info. # (2) Let the server set the CSRF cookie. # No authentication has been set up yet, so these requests will be unauthenticated. # https://cheatsheetseries.owasp.org/cheatsheets/Cross-Site_Request_Forgery_Prevention_Cheat_Sheet.html#double-submit-cookie if offline: self.server_info = self.get_json(self.api_uri) else: # We need a CSRF token. with self.disable_cache(allow_read=False, allow_write=True): self.server_info = self.get_json(self.api_uri) self.api_key = api_key # property setter sets Authorization header
def __getstate__(self): if getattr(self.http_client, "app", None): raise TypeError( "Cannot pickle a Tiled Context built around an ASGI. " "Only Tiled Context connected to remote servers can be pickled." ) return ( tiled_version, self.api_uri, self.http_client.headers, list(self.http_client.cookies.jar), self.http_client.timeout, self.http_client.auth, self._verify, self.offline, self._revalidate, self._token_cache, self.server_info, self.cache, ) def __setstate__(self, state): ( state_tiled_version, api_uri, headers, cookies_list, timeout, auth, verify, offline, revalidate, token_cache, server_info, cache, ) = state if state_tiled_version != tiled_version: raise TypeError( f"Cannot unpickle {type(self).__name__} from tiled version {state_tiled_version} " f"using tiled version {tiled_version}. Pickle should only be used to short-term " "transfer between identical versions of tiled." ) self.api_uri = api_uri cookies = httpx.Cookies() for cookie in cookies_list: cookies.set( cookie.name, cookie.value, domain=cookie.domain, path=cookie.path ) self.http_client = httpx.Client( verify=verify, event_hooks=EVENT_HOOKS, cookies=cookies, timeout=timeout, headers=headers, follow_redirects=True, auth=auth, ) self._revalidate = revalidate self._token_cache = token_cache self._cache = cache self.server_info = server_info self.offline = offline @classmethod def from_any_uri( cls, uri, *, headers=None, api_key=None, cache=None, offline=False, timeout=None, verify=True, token_cache=DEFAULT_TOKEN_CACHE, app=None, ): """ Accept a URI to a specific node. For example, given URI "https://example.com/api/v1/node/metadata/a/b/c" return a Context connected to "https://examples/api/v1" and the list ["a", "b", "c"]. """ uri = httpx.URL(uri) node_path_parts = [] if "/node/metadata" in uri.path: api_path, _, node_path = uri.path.partition("/node/metadata") api_uri = uri.copy_with(path=api_path) node_path_parts.extend( [segment for segment in node_path.split("/") if segment] ) # Below we handle the case where we are given *less* than the full URI # to the root endpoint. Here we are taking some care to plan for the case # where tiled is served at a sub-path, even though that is not yet supported # on the server side. elif "/api" not in uri.path: # Looks like we were given the root path (to the HTML landing page). path = uri.path if path.endswith("/"): path = path[:-1] api_uri = uri.copy_with(path=f"{path}/api/v1") elif "/v1" not in uri.path: # Looks like we were given the /api but no version. path = uri.path if path.endswith("/"): path = path[:-1] api_uri = uri.copy_with(path=f"{path}/v1") else: api_uri = uri context = cls( api_uri, headers=headers, api_key=api_key, cache=cache, offline=offline, timeout=timeout, verify=verify, token_cache=token_cache, app=app, ) return context, node_path_parts @property def tokens(self): "A view of the current access and refresh tokens." return DictView(self.http_client.auth.tokens) @property def api_key(self): # Extract from header to ensure that there is one "ground truth" here # and no possibility of state getting out of sync. header = self.http_client.headers.get("Authorization", "") match = API_KEY_AUTH_HEADER_PATTERN.match(header) if match is not None: return match.group(1) @api_key.setter def api_key(self, api_key): if api_key is None: if self.http_client.headers.get("Authorization", "").startswith("Apikey "): self.http_client.headers.pop("Authorization") else: self.http_client.headers["Authorization"] = f"Apikey {api_key}" @property def cache(self): return self._cache @property def offline(self): return self._offline @offline.setter def offline(self, value): offline = bool(value) if offline and (self._cache is None): raise RuntimeError( """To use offline mode, Tiled must be configured with a Cache, as in >>> from tiled.client import from_uri >>> from tiled.client.cache import Cache >>> client = from_uri("...", cache=Cache.on_disk("my_cache")) """ ) self._offline = offline if not self._offline: # We need a CSRF token. with self.disable_cache(allow_read=False, allow_write=True): self.server_info = self.get_json(self.api_uri) def which_api_key(self): """ A 'who am I' for API keys """ if not self.api_key: raise RuntimeError("Not API key is configured for the client.") return self.get_json(self.server_info["authentication"]["links"]["apikey"]) def create_api_key(self, scopes=None, expires_in=None, note=None): """ Generate a new API for the currently-authenticated user. Parameters ---------- scopes : Optional[List[str]] Restrict the access available to the API key by listing specific scopes. By default, this will have the same access as the user. expires_in : Optional[int] Number of seconds until API key expires. If None, it will never expire or it will have the maximum lifetime allowed by the server. note : Optional[str] Description (for humans). """ return self.post_json( self.server_info["authentication"]["links"]["apikey"], {"scopes": scopes, "expires_in": expires_in, "note": note}, ) def revoke_api_key(self, first_eight): request = self.http_client.build_request( "DELETE", self.server_info["authentication"]["links"]["apikey"], headers={"x-csrf": self.http_client.cookies["tiled_csrf"]}, params={"first_eight": first_eight}, ) response = self.http_client.send(request) handle_error(response) @property def app(self): warnings.warn( "The 'app' accessor on Context is deprecated. Use Context.http_client.app.", DeprecationWarning, stacklevel=2, ) return self.http_client.app @property def base_url(self): warnings.warn( "The 'base_url' accessor on Context is deprecated. Use Context.http_client.base_url.", DeprecationWarning, stacklevel=2, ) return self.http_client.base_url @property def event_hooks(self): "httpx.Client event hooks. This is exposed for testing." warnings.warn( "The 'event_hooks' accessor on Context is deprecated. Use Context.http_client.event_hooks.", DeprecationWarning, stacklevel=2, ) return self.http_client.event_hooks @property def revalidate(self): """ This controls how aggressively to check whether cache entries are out of date. - FORCE: Always revalidate (generally too aggressive and expensive) - IF_EXPIRED: Revalidate if the "Expire" date provided by the server has passed - IF_WE_MUST: Only revalidate if the server indicated that is is a particularly volatile entry, such as a search result to a dynamic query. """ return self._revalidate @revalidate.setter def revalidate(self, value): self._revalidate = Revalidate(value) @contextlib.contextmanager def revalidation(self, revalidate): """ Temporarily change the 'revalidate' property in a context. Parameters ---------- revalidate: string or tiled.client.cache.Revalidate enum member """ try: member = Revalidate(revalidate) except ValueError as err: # This raises a more helpful error that lists the valid options. raise ValueError( f"Revalidation {revalidate} not recognized. Must be one of {set(Revalidate.__members__)}" ) from err original = self.revalidate self.revalidate = member yield # Upon leaving context, set it back. self.revalidate = original @contextlib.contextmanager def disable_cache(self, allow_read=False, allow_write=False): self._disable_cache_read = not allow_read self._disable_cache_write = not allow_write yield self._disable_cache_read = False self._disable_cache_write = False def get_content(self, path, accept=None, stream=False, revalidate=None, **kwargs): send_kwargs = {} if "auth" in kwargs: send_kwargs["auth"] = kwargs.pop("auth") if revalidate is None: # Fallback to context default. revalidate = self.revalidate request = self.http_client.build_request("GET", path, **kwargs) if accept: request.headers["Accept"] = accept url = request.url if self._offline: # We must rely on the cache alone. # The role of a 'reservation' is to ensure that the content # of interest is not evicted from the cache between the moment # that we start verifying its validity and the moment that # we actually read the content. It is used more extensively # below. reservation = self._cache.get_reservation(url) if reservation is None: raise NotAvailableOffline(url) content = reservation.load_content() if content is None: # TODO Do we ever get here? raise NotAvailableOffline(url) return content if self._cache is None: # No cache, so we can use the client straightforwardly. response = self.http_client.send(request, stream=stream) handle_error(response) if response.headers.get("content-encoding") == "blosc": import blosc return blosc.decompress(response.content) return response.content # If we get this far, we have an online client and a cache. # Parse Cache-Control header directives. cache_control = { directive.lstrip(" ") for directive in request.headers.get("Cache-Control", "").split(",") } if "no-cache" in cache_control: reservation = None else: reservation = self._cache.get_reservation(url) try: if reservation is not None: is_stale = reservation.is_stale() if not ( # This condition means "client user wants us to unconditionally revalidate" (revalidate == Revalidate.FORCE) or # This condition means "client user wants us to revalidate if expired" (is_stale and (revalidate == Revalidate.IF_EXPIRED)) or # This condition means "server really wants us to revalidate" (is_stale and reservation.item.must_revalidate) or self._disable_cache_read ): # Short-circuit. Do not even bother consulting the server. return reservation.load_content() if not self._disable_cache_read: request.headers["If-None-Match"] = reservation.item.etag response = self.http_client.send(request, stream=stream, **send_kwargs) handle_error(response) if response.status_code == 304: # HTTP 304 Not Modified # Update the expiration time. reservation.renew(response.headers.get("expires")) # Read from the cache return reservation.load_content() elif not response.is_error: etag = response.headers.get("ETag") encoding = response.headers.get("Content-Encoding") content = response.content # httpx handles standard HTTP encodings transparently, but we have to # handle "blosc" manually. if encoding == "blosc": import blosc content = blosc.decompress(content) if ( ("no-store" not in cache_control) and (etag is not None) and (not self._disable_cache_write) ): # Write to cache. self._cache.put( url, response.headers, content, ) return content else: raise NotImplementedError( f"Unexpected status_code {response.status_code}" ) finally: if reservation is not None: reservation.ensure_released() def get_json(self, path, stream=False, **kwargs): return msgpack.unpackb( self.get_content( path, accept="application/x-msgpack", stream=stream, **kwargs ), timestamp=3, # Decode msgpack Timestamp as datetime.datetime object. ) def post_json(self, path, content): request = self.http_client.build_request( "POST", path, json=content, # Submit CSRF token in both header and cookie. # https://cheatsheetseries.owasp.org/cheatsheets/Cross-Site_Request_Forgery_Prevention_Cheat_Sheet.html#double-submit-cookie headers={ "x-csrf": self.http_client.cookies["tiled_csrf"], "accept": "application/x-msgpack", }, ) response = self.http_client.send(request) handle_error(response) return msgpack.unpackb( response.content, timestamp=3, # Decode msgpack Timestamp as datetime.datetime object. ) def put_json(self, path, content): request = self.http_client.build_request( "PUT", path, json=content, # Submit CSRF token in both header and cookie. # https://cheatsheetseries.owasp.org/cheatsheets/Cross-Site_Request_Forgery_Prevention_Cheat_Sheet.html#double-submit-cookie headers={ "x-csrf": self.http_client.cookies["tiled_csrf"], "accept": "application/x-msgpack", }, ) response = self.http_client.send(request) handle_error(response) return msgpack.unpackb( response.content, timestamp=3, # Decode msgpack Timestamp as datetime.datetime object. ) def put_content(self, path, content, headers=None, params=None): # Submit CSRF token in both header and cookie. # https://cheatsheetseries.owasp.org/cheatsheets/Cross-Site_Request_Forgery_Prevention_Cheat_Sheet.html#double-submit-cookie headers = headers or {} headers.setdefault("x-csrf", self.http_client.cookies["tiled_csrf"]) headers.setdefault("accept", "application/x-msgpack") request = self.http_client.build_request( "PUT", path, content=content, headers=headers, params=params, ) response = self.http_client.send(request) handle_error(response) return msgpack.unpackb( response.content, timestamp=3, # Decode msgpack Timestamp as datetime.datetime object. ) def delete_content(self, path, content, headers=None, params=None): # Submit CSRF token in both header and cookie. # https://cheatsheetseries.owasp.org/cheatsheets/Cross-Site_Request_Forgery_Prevention_Cheat_Sheet.html#double-submit-cookie headers = headers or {} headers.setdefault("x-csrf", self.http_client.cookies["tiled_csrf"]) headers.setdefault("accept", "application/x-msgpack") request = self.http_client.build_request( "DELETE", path, content=None, headers=headers, params=params ) response = self.http_client.send(request) handle_error(response) return msgpack.unpackb( response.content, timestamp=3, # Decode msgpack Timestamp as datetime.datetime object. )
[docs] def authenticate(self, username=None, provider=None): """ See login. This is for programmatic use. """ providers = self.server_info["authentication"]["providers"] spec = _choose_identity_provider(providers, provider) provider = spec["provider"] if self.api_key is not None: # Check that API key authenticates us as this user, # and then either return or raise. identities = self.whoami()["identities"] for identity in identities: if (identity["provider"] == provider) and (identity["id"] == username): return raise RuntimeError( "An API key is set, and it is not associated with the username/provider " f"{username}/{provider}. Unset the API key first." ) refresh_url = self.server_info["authentication"]["links"]["refresh_session"] csrf_token = self.http_client.cookies["tiled_csrf"] # If we are passed a username, we can check whether we already have # tokens stashed. if username is not None: token_directory = self._token_directory(provider, username) self.http_client.auth = TiledAuth(refresh_url, csrf_token, token_directory) # This will either: # * Use an access_token and succeed. # * Use a refresh_token to attempt refresh flow and succeed. # * Use a refresh_token to attempt refresh flow and fail, raise. # * Find no tokens and raise. try: self.whoami() except CannotRefreshAuthentication: # Continue below, where we will prompt for log in. self.http_client.auth = None else: # We have a live session for the specified provider and username already. # No need to log in again. return self.http_client.auth = None mode = spec["mode"] auth_endpoint = spec["links"]["auth_endpoint"] if mode == "password": if username: print(f"Username {username}") else: username = input("Username: ") password = getpass.getpass() form_data = { "grant_type": "password", "username": username, "password": password, } token_response = self.http_client.post( auth_endpoint, data=form_data, auth=None ) handle_error(token_response) tokens = token_response.json() elif mode == "external": verification_response = self.http_client.post( auth_endpoint, json={}, auth=None ) handle_error(verification_response) verification = verification_response.json() authorization_uri = verification["authorization_uri"] print( f""" You have {int(verification['expires_in']) // 60} minutes visit this URL {authorization_uri} and enter the code: {verification['user_code']} """ ) import webbrowser webbrowser.open(authorization_uri) deadline = verification["expires_in"] + time.monotonic() print("Waiting...", end="", flush=True) while True: time.sleep(verification["interval"]) if time.monotonic() > deadline: raise Exception("Deadline expired.") access_response = self.http_client.post( verification["verification_uri"], json={ "device_code": verification["device_code"], "grant_type": "urn:ietf:params:oauth:grant-type:device_code", }, auth=None, ) if (access_response.status_code == 400) and ( access_response.json()["detail"]["error"] == "authorization_pending" ): print(".", end="", flush=True) continue handle_error(access_response) print("") break tokens = access_response.json() else: raise ValueError(f"Server has unknown authentication mechanism {mode!r}") username = tokens["identity"]["id"] token_directory = self._token_directory(provider, username) auth = TiledAuth(refresh_url, csrf_token, token_directory) auth.sync_set_token("access_token", tokens["access_token"]) auth.sync_set_token("refresh_token", tokens["refresh_token"]) self.http_client.auth = auth confirmation_message = spec.get("confirmation_message") if confirmation_message: print(confirmation_message.format(id=username)) return spec, username
[docs] def login(self, username=None, provider=None): """ Depending on the server's authentication method, this will prompt for username/password: >>> c.login() Username: USERNAME Password: <input is hidden> or prompt you to open a link in a web browser to login with a third party: >>> c.login() You have ... minutes visit this URL https://... and enter the code: XXXX-XXXX """ self.authenticate(username, provider) # For programmatic access to the return values, use authenticate(). # This returns None in order to provide a clean UX in an interpreter. return None
def _token_directory(self, provider, username): # ~/.config/tiled/tokens/{host:port}/{provider}/{username} # with each templated element URL-encoded so it is a valid filename. return Path( self._token_cache, urllib.parse.quote_plus(str(self.api_uri)), urllib.parse.quote_plus(provider), urllib.parse.quote_plus(username), )
[docs] def force_auth_refresh(self): """ Execute refresh flow. This method is exposed for testing and debugging uses. It should never be necessary for the user to call. Refresh flow is automatically executed by tiled.client.auth.TiledAuth when the current access_token expires. """ if self.http_client.auth is None: raise RuntimeError( "No authentication has been set up. Cannot reauthenticate." ) refresh_token = self.http_client.auth.sync_get_token( "refresh_token", reload_from_disk=True ) if refresh_token is None: raise CannotRefreshAuthentication("There is no refresh_token.") csrf_token = self.http_client.cookies["tiled_csrf"] refresh_request = build_refresh_request( self.http_client.auth.refresh_url, refresh_token, csrf_token, ) token_response = self.http_client.send(refresh_request, auth=None) if token_response.status_code == 401: raise CannotRefreshAuthentication( "Session cannot be refreshed. Log in again." ) handle_error(token_response) tokens = token_response.json() self.http_client.auth.sync_set_token("access_token", tokens["access_token"]) self.http_client.auth.sync_set_token("refresh_token", tokens["refresh_token"]) return tokens
def whoami(self): "Return information about the currently-authenticated user or service." return self.get_json(self.server_info["authentication"]["links"]["whoami"])
[docs] def logout(self): """ Log out of the current session (if any). This method is idempotent. """ if self.http_client.auth is None: return # Revoke the current session. self.http_client.post(f"{self.api_uri}auth/logout") # Clear on-disk state. self.http_client.auth.sync_clear_token("access_token") self.http_client.auth.sync_clear_token("refresh_token") # Clear in-memory state. self.http_client.headers.pop("Authorization", None) self.http_client.auth = None
def revoke_session(self, session_id): """ Revoke a Session so it cannot be refreshed. This may be done to ensure that a possibly-leaked refresh token cannot be used. """ request = self.http_client.build_request( "DELETE", self.server_info["authentication"]["links"]["revoke_session"].format( session_id=session_id ), headers={"x-csrf": self.http_client.cookies["tiled_csrf"]}, ) response = self.http_client.send(request) handle_error(response)
def _choose_identity_provider(providers, provider=None): if provider is not None: for spec in providers: if spec["provider"] == provider: break else: raise ValueError( f"No such provider {provider}. Choices are {[spec['provider'] for spec in providers]}" ) else: if len(providers) == 1: # There is only one choice, so no need to prompt the user. (spec,) = providers else: while True: print("Authenticaiton providers:") for i, spec in enumerate(providers, start=1): print(f"{i} - {spec['provider']}") raw_choice = input( "Choose an authentication provider (or press Enter to escape): " ) if not raw_choice: print("No authentication provider chosen. Failed.") break try: choice = int(raw_choice) except TypeError: print("Choice must be a number.") continue try: spec = providers[choice - 1] except IndexError: print(f"Choice must be a number 1 through {len(providers)}.") continue break return spec