From 902762175a0c6f6fb797d46fa4deb393a2cf2e0e Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Mon, 6 Nov 2023 10:34:47 +0100 Subject: [PATCH] Refactor response flag parsing ## Motivation / Description We spent a lot of time tokenizing and then parsing the response flags. This does the flag parsing in one single loop, speeding things up. Also this rewrites response flags not to be Flags/IntFlags as they can't be used for requests, and having them as part of the Value dataclass speeds things up as well as simplifies the usage of the library. ## Changes introduced - Avoid header tokenization phase - Move flags to Success/Value class as attributes - Move flag parsing to those classes. --- .../commands/high_level_commands.py | 16 +- .../connection/memcache_socket.py | 106 +++-------- src/meta_memcache/executors/default.py | 4 +- .../extras/migrating_cache_client.py | 2 +- .../extras/probabilistic_hot_cache.py | 6 +- src/meta_memcache/protocol.py | 174 +++++++++++++++--- tests/commands_test.py | 114 +++++------- tests/memcache_socket_test.py | 22 +-- tests/migrating_cache_client_test.py | 8 +- tests/probabilistic_hot_cache_test.py | 18 +- 10 files changed, 256 insertions(+), 214 deletions(-) diff --git a/src/meta_memcache/commands/high_level_commands.py b/src/meta_memcache/commands/high_level_commands.py index 790b936..96a47ec 100644 --- a/src/meta_memcache/commands/high_level_commands.py +++ b/src/meta_memcache/commands/high_level_commands.py @@ -289,12 +289,11 @@ def get_or_lease_cas( if isinstance(result, Value): # It is a hit. - cas_token = result.int_flags.get(IntFlag.RETURNED_CAS_TOKEN) - if Flag.WIN in result.flags: + if result.win: # Win flag present, meaning we got the lease to # recache/cache the item. We need to mimic a miss. - return None, cas_token - if result.size == 0 and Flag.LOST in result.flags: + return None, result.cas_token + if result.size == 0 and result.win is False: # The value is empty, this is a miss lease, # and we lost, so we must keep retrying and # wait for the winner to populate the value. @@ -302,11 +301,11 @@ def get_or_lease_cas( continue else: # We run out of retries, behave as a miss - return None, cas_token + return None, result.cas_token else: # There is data, either the is no lease or # we lost and should use the stale value. - return result.value, cas_token + return result.value, result.cas_token else: # With MISS_LEASE_TTL we should always get a value # because on miss a lease empty value is generated @@ -381,8 +380,7 @@ def get_cas( if result is None: return None, None else: - cas_token = result.int_flags.get(IntFlag.RETURNED_CAS_TOKEN) - return result.value, cas_token + return result.value, result.cas_token def _get( self: HighLevelCommandMixinWithMetaCommands, @@ -418,7 +416,7 @@ def _process_get_result( ) -> Optional[Value]: if isinstance(result, Value): # It is a hit - if Flag.WIN in result.flags: + if result.win: # Win flag present, meaning we got the lease to # recache the item. We need to mimic a miss, so # we set the value to None. diff --git a/src/meta_memcache/connection/memcache_socket.py b/src/meta_memcache/connection/memcache_socket.py index 7bf14cf..4c1d967 100644 --- a/src/meta_memcache/connection/memcache_socket.py +++ b/src/meta_memcache/connection/memcache_socket.py @@ -1,23 +1,19 @@ import logging import socket -from typing import Iterable, List, Union +from typing import Union from meta_memcache.errors import MemcacheError from meta_memcache.protocol import ( ENDL, ENDL_LEN, NOOP, - SPACE, Conflict, Miss, NotStored, ServerVersion, Success, Value, - flag_values, get_store_success_response_header, - int_flags_values, - token_flags_values, ) _log: logging.Logger = logging.getLogger(__name__) @@ -119,73 +115,34 @@ def _reset_buffer(self) -> None: Reset buffer moving remaining bytes in it (if any) """ remaining_data = self._read - self._pos - if remaining_data > 0: - if self._pos <= self._reset_buffer_size: - # Avoid moving memory if buffer still has - # spare capacity for new responses. If the - # whole buffer us used, we will just reset - # the pointers and save a lot of memory - # data copies - return - self._buf_view[0:remaining_data] = self._buf_view[self._pos : self._read] + self._buf_view[0:remaining_data] = self._buf_view[self._pos : self._read] self._pos = 0 self._read = remaining_data - def _recv_header(self) -> memoryview: - endl_pos = self._buf.find(ENDL, self._pos, self._read) - while endl_pos < 0 and self._read < self._buffer_size: + def _get_single_header(self) -> memoryview: + # Reset buffer for new data + if self._read == self._pos: + self._read = 0 + self._pos = 0 + elif self._pos > self._reset_buffer_size: + self._reset_buffer() + + endl_pos = -1 + while True: + if self._read - self._pos > ENDL_LEN: + endl_pos = self._buf.find(ENDL, self._pos, self._read) + if endl_pos >= 0: + break # Missing data, but still space in buffer, so read more if self._recv_info_buffer() <= 0: break - endl_pos = self._buf.find(ENDL, self._pos, self._read) if endl_pos < 0: raise MemcacheError("Bad response. Socket might have closed unexpectedly") - header = self._buf_view[self._pos : endl_pos] + pos = self._pos self._pos = endl_pos + ENDL_LEN - return header - - def _add_flags(self, success: Success, chunks: Iterable[memoryview]) -> None: - """ - Each flag starts with one byte for the flag, and and optional int/byte - value depending on the flag. - """ - for chunk in chunks: - flag = chunk[0] - if len(chunk) == 1: - # Flag without value - if f := flag_values.get(flag): - success.flags.add(f) - else: - _log.warning(f"Unrecognized flag {bytes(chunk)!r}") - else: - # Value flag - if int_flag := int_flags_values.get(flag): - success.int_flags[int_flag] = int(chunk[1:]) - elif token_flag := token_flags_values.get(flag): - success.token_flags[token_flag] = bytes(chunk[1:]) - else: - _log.warning(f"Unrecognized flag {bytes(chunk)!r}") - - def _tokenize_header(self, header: memoryview) -> List[memoryview]: - """ - Slice header by spaces into memoryview chunks - """ - chunks = [] - prev, i = 0, -1 - for i, v in enumerate(header): - if v == SPACE: - if i > prev: - chunks.append(header[prev:i]) - prev = i + 1 - if prev <= i: - chunks.append(header[prev:]) - return chunks - - def _get_single_header(self) -> List[memoryview]: - self._reset_buffer() - return self._tokenize_header(self._recv_header()) + return self._buf_view[pos:endl_pos] def sendall(self, data: bytes, with_noop: bool = False) -> None: if with_noop: @@ -195,11 +152,11 @@ def sendall(self, data: bytes, with_noop: bool = False) -> None: def _read_until_noop_header(self) -> None: while self._noop_expected > 0: - response_code, *_chunks = self._get_single_header() - if response_code == b"MN": + header = self._get_single_header() + if header[0:2] == b"MN": self._noop_expected -= 1 - def _get_header(self) -> List[memoryview]: + def _get_header(self) -> memoryview: try: if self._noop_expected > 0: self._read_until_noop_header() @@ -212,37 +169,30 @@ def _get_header(self) -> List[memoryview]: def get_response( self, ) -> Union[Value, Success, NotStored, Conflict, Miss]: - header = self._get_header() + header = self._get_header().tobytes() + response_code = header[0:2] result: Union[Value, Success, NotStored, Conflict, Miss] try: - response_code, *chunks = header if response_code == b"VA": - # Value response, parse size and flags - value_size = int(chunks.pop(0)) - result = Value(value_size) - self._add_flags(result, chunks) + # Value response + result = Value.from_header(header) elif response_code == self._store_success_response_header: # Stored or no value, return Success - result = Success() - self._add_flags(result, chunks) + result = Success.from_header(header) elif response_code == b"NS": # Value response, parse size and flags result = NOT_STORED - assert len(chunks) == 0 # noqa: S101 elif response_code == b"EX": # Already exists, not changed, CAS conflict result = CONFLICT - assert len(chunks) == 0 # noqa: S101 elif response_code == b"EN" or response_code == b"NF": # Not Found, Miss. result = MISS - assert len(chunks) == 0 # noqa: S101 else: raise MemcacheError(f"Unknown response: {bytes(response_code)!r}") except Exception as e: - response = b" ".join(header).decode() - _log.warning(f"Error parsing response header in {self}: {response}") - raise MemcacheError(f"Error parsing response header {response}") from e + _log.warning(f"Error parsing response header in {self}: {header!r}") + raise MemcacheError(f"Error parsing response header {header!r}") from e return result diff --git a/src/meta_memcache/executors/default.py b/src/meta_memcache/executors/default.py index bba5e3b..6e382ae 100644 --- a/src/meta_memcache/executors/default.py +++ b/src/meta_memcache/executors/default.py @@ -252,12 +252,12 @@ def _conn_recv_response( Read response on a connection """ if flags and Flag.NOREPLY in flags: - return Success(flags=set([Flag.NOREPLY])) + return Success() result = conn.get_response() if isinstance(result, Value): data = conn.get_value(result.size) if result.size > 0: - encoding_id = result.int_flags.get(IntFlag.CLIENT_FLAG, 0) + encoding_id = result.client_flag or 0 try: result.value = self._serializer.unserialize(data, encoding_id) except Exception: diff --git a/src/meta_memcache/extras/migrating_cache_client.py b/src/meta_memcache/extras/migrating_cache_client.py index f62fcb1..b1ce0d1 100644 --- a/src/meta_memcache/extras/migrating_cache_client.py +++ b/src/meta_memcache/extras/migrating_cache_client.py @@ -75,7 +75,7 @@ def get_migration_mode(self) -> MigrationMode: return current_mode def _get_value_ttl(self, value: Value) -> int: - ttl = value.int_flags.get(IntFlag.TTL, self._default_read_backfill_ttl) + ttl = value.ttl if value.ttl is not None else self._default_read_backfill_ttl if ttl < 0: # TTL for items marked to store forvered is returned as -1 ttl = 0 diff --git a/src/meta_memcache/extras/probabilistic_hot_cache.py b/src/meta_memcache/extras/probabilistic_hot_cache.py index e2704c9..1e69699 100644 --- a/src/meta_memcache/extras/probabilistic_hot_cache.py +++ b/src/meta_memcache/extras/probabilistic_hot_cache.py @@ -10,7 +10,7 @@ from meta_memcache.extras.client_wrapper import ClientWrapper from meta_memcache.interfaces.cache_api import CacheApi from meta_memcache.metrics.base import BaseMetricsCollector, MetricDefinition -from meta_memcache.protocol import IntFlag, Key, Value +from meta_memcache.protocol import Key, Value @dataclass @@ -124,8 +124,8 @@ def _store_in_hot_cache_if_necessary( allowed: bool, ) -> None: if not is_hot: - hit_after_write = value.int_flags.get(IntFlag.HIT_AFTER_WRITE, 0) - last_read_age = value.int_flags.get(IntFlag.LAST_READ_AGE, 9999) + hit_after_write = value.fetched or 0 + last_read_age = value.last_access if value.last_access is not None else 9999 if ( hit_after_write > 0 and last_read_age <= self._max_last_access_age_seconds diff --git a/src/meta_memcache/protocol.py b/src/meta_memcache/protocol.py index 831c7a6..95216d0 100644 --- a/src/meta_memcache/protocol.py +++ b/src/meta_memcache/protocol.py @@ -1,6 +1,6 @@ from dataclasses import dataclass from enum import Enum, IntEnum -from typing import Any, Dict, List, Optional, Set, Union +from typing import Any, Dict, List, Optional, Union ENDL = b"\r\n" NOOP: bytes = b"mn" + ENDL @@ -56,30 +56,21 @@ class Flag(Enum): RETURN_FETCHED = b"h" RETURN_KEY = b"k" NO_UPDATE_LRU = b"u" - WIN = b"W" - LOST = b"Z" - STALE = b"X" MARK_STALE = b"I" class IntFlag(Enum): - TTL = b"t" CACHE_TTL = b"T" RECACHE_TTL = b"R" MISS_LEASE_TTL = b"N" - CLIENT_FLAG = b"f" SET_CLIENT_FLAG = b"F" - LAST_READ_AGE = b"l" - HIT_AFTER_WRITE = b"h" MA_INITIAL_VALUE = b"J" MA_DELTA_VALUE = b"D" - RETURNED_CAS_TOKEN = b"c" CAS_TOKEN = b"C" class TokenFlag(Enum): OPAQUE = b"O" - KEY = b"k" # 'M' (mode switch): # * Meta Arithmetic: # - I or +: increment @@ -108,42 +99,171 @@ class MemcacheResponse: class Miss(MemcacheResponse): __slots__ = () + pass + +# Response flags +TOKEN_FLAG_OPAQUE = ord("O") +INT_FLAG_CAS_TOKEN = ord("c") +INT_FLAG_FETCHED = ord("h") +INT_FLAG_LAST_ACCESS = ord("l") +INT_FLAG_TTL = ord("t") +INT_FLAG_CLIENT_FLAG = ord("f") +INT_FLAG_SIZE = ord("s") +FLAG_WIN = ord("W") +FLAG_LOST = ord("Z") +FLAG_STALE = ord("X") + + +# @dataclass(slots=True, init=False) @dataclass class Success(MemcacheResponse): - __slots__ = ("flags", "int_flags", "token_flags") - flags: Set[Flag] - int_flags: Dict[IntFlag, int] - token_flags: Dict[TokenFlag, bytes] + __slots__ = ( + "cas_token", + "fetched", + "last_access", + "ttl", + "client_flag", + "win", + "stale", + "real_size", + "opaque", + ) + cas_token: Optional[int] + fetched: Optional[int] + last_access: Optional[int] + ttl: Optional[int] + client_flag: Optional[int] + win: Optional[bool] + stale: bool + real_size: Optional[int] + opaque: Optional[bytes] def __init__( self, - flags: Optional[Set[Flag]] = None, - int_flags: Optional[Dict[IntFlag, int]] = None, - token_flags: Optional[Dict[TokenFlag, bytes]] = None, + *, + cas_token: Optional[int] = None, + fetched: Optional[int] = None, + last_access: Optional[int] = None, + ttl: Optional[int] = None, + client_flag: Optional[int] = None, + win: Optional[bool] = None, + stale: bool = False, + real_size: Optional[int] = None, + opaque: Optional[bytes] = None, ) -> None: - self.flags = flags or set() - self.int_flags = int_flags or {} - self.token_flags = token_flags or {} - - + self.cas_token = cas_token + self.fetched = fetched + self.last_access = last_access + self.ttl = ttl + self.client_flag = client_flag + self.win = win + self.stale = stale + self.real_size = real_size + self.opaque = opaque + + @classmethod + def from_header(cls, header: "Blob") -> "Success": + result = cls() + result._set_flags(header) + return result + + def _set_flags(self, header: bytes, pos: int = 3) -> None: # noqa: C901 + header_size = len(header) + while pos < header_size: + flag = header[pos] + pos += 1 + if flag == SPACE: + continue + end = pos + while end < header_size: + if header[end] == SPACE: + break + end += 1 + + if flag == INT_FLAG_CAS_TOKEN: + self.cas_token = int(header[pos:end]) + elif flag == INT_FLAG_FETCHED: + self.fetched = int(header[pos:end]) + elif flag == INT_FLAG_LAST_ACCESS: + self.last_access = int(header[pos:end]) + elif flag == INT_FLAG_TTL: + self.ttl = int(header[pos:end]) + elif flag == INT_FLAG_CLIENT_FLAG: + self.client_flag = int(header[pos:end]) + elif flag == FLAG_WIN: + self.win = True + elif flag == FLAG_LOST: + self.win = False + elif flag == FLAG_STALE: + self.stale = True + elif flag == INT_FLAG_SIZE: + self.real_size = int(header[pos:end]) + elif flag == TOKEN_FLAG_OPAQUE: + self.opaque = header[pos:end] + pos = end + 1 + + +# @dataclass(slots=True, init=False) @dataclass class Value(Success): - __slots__ = ("flags", "int_flags", "token_flags", "size", "value") + __slots__ = ( + "cas_token", + "fetched", + "last_access", + "ttl", + "client_flag", + "win", + "stale", + "real_size", + "opaque", + "size", + "value", + ) size: int value: Optional[Any] def __init__( self, + *, size: int, value: Optional[Any] = None, - flags: Optional[Set[Flag]] = None, - int_flags: Optional[Dict[IntFlag, int]] = None, - token_flags: Optional[Dict[TokenFlag, bytes]] = None, + cas_token: Optional[int] = None, + fetched: Optional[int] = None, + last_access: Optional[int] = None, + ttl: Optional[int] = None, + client_flag: Optional[int] = None, + win: Optional[bool] = None, + stale: bool = False, + real_size: Optional[int] = None, + opaque: Optional[bytes] = None, ) -> None: - super().__init__(flags, int_flags, token_flags) self.size = size self.value = value + self.cas_token = cas_token + self.fetched = fetched + self.last_access = last_access + self.ttl = ttl + self.client_flag = client_flag + self.win = win + self.stale = stale + self.real_size = real_size + self.opaque = opaque + + @classmethod + def from_header(cls, header: "Blob") -> "Value": + header_size = len(header) + if header_size < 4 or header[2] != SPACE: + raise ValueError(f"Invalid header {header!r}") + end = 4 + while end < header_size: + if header[end] == SPACE: + break + end += 1 + size = int(header[3:end]) + result = cls(size=size) + result._set_flags(header, pos=end + 1) + return result @dataclass diff --git a/tests/commands_test.py b/tests/commands_test.py index 5d5ac1a..a86aee6 100644 --- a/tests/commands_test.py +++ b/tests/commands_test.py @@ -25,8 +25,6 @@ from meta_memcache.interfaces.cache_api import CacheApi from meta_memcache.interfaces.meta_commands import MetaCommandsProtocol from meta_memcache.protocol import ( - Flag, - IntFlag, Miss, NotStored, ServerVersion, @@ -565,7 +563,7 @@ def test_get_cmd(memcache_socket: MemcacheSocket, cache_client: CacheClient) -> ) memcache_socket.sendall.reset_mock() - memcache_socket.get_response.return_value = Value(size=0) + memcache_socket.get_response.return_value = Value(size=0, value=None) cache_client.get_or_lease( key=Key("foo"), lease_policy=LeasePolicy(), @@ -618,10 +616,9 @@ def test_get_value(memcache_socket: MemcacheSocket, cache_client: CacheClient) - encoded_value = MixedSerializer().serialize(expected_value) memcache_socket.get_response.return_value = Value( size=len(encoded_value.data), - int_flags={ - IntFlag.CLIENT_FLAG: encoded_value.encoding_id, - IntFlag.RETURNED_CAS_TOKEN: expected_cas_token, - }, + value=None, + cas_token=expected_cas_token, + client_flag=encoded_value.encoding_id, ) memcache_socket.get_value.return_value = encoded_value.data @@ -662,10 +659,9 @@ def test_value_wrong_type( encoded_value = MixedSerializer().serialize(expected_value) memcache_socket.get_response.return_value = Value( size=len(encoded_value.data), - int_flags={ - IntFlag.CLIENT_FLAG: encoded_value.encoding_id, - IntFlag.RETURNED_CAS_TOKEN: expected_cas_token, - }, + value=None, + cas_token=expected_cas_token, + client_flag=encoded_value.encoding_id, ) memcache_socket.get_value.return_value = encoded_value.data @@ -697,10 +693,9 @@ def test_deserialization_error( encoded_value = EncodedValue(data=b"invalid", encoding_id=MixedSerializer.PICKLE) memcache_socket.get_response.return_value = Value( size=len(encoded_value.data), - int_flags={ - IntFlag.CLIENT_FLAG: encoded_value.encoding_id, - IntFlag.RETURNED_CAS_TOKEN: expected_cas_token, - }, + value=None, + cas_token=expected_cas_token, + client_flag=encoded_value.encoding_id, ) memcache_socket.get_value.return_value = encoded_value.data @@ -723,11 +718,11 @@ def test_recache_win_returns_miss( encoded_value = MixedSerializer().serialize(expected_value) memcache_socket.get_response.return_value = Value( size=len(encoded_value.data), - flags=set([Flag.WIN, Flag.STALE]), - int_flags={ - IntFlag.CLIENT_FLAG: encoded_value.encoding_id, - IntFlag.RETURNED_CAS_TOKEN: expected_cas_token, - }, + value=None, + win=True, + stale=True, + cas_token=expected_cas_token, + client_flag=encoded_value.encoding_id, ) memcache_socket.get_value.return_value = encoded_value.data @@ -744,11 +739,11 @@ def test_recache_lost_returns_stale_value( encoded_value = MixedSerializer().serialize(expected_value) memcache_socket.get_response.return_value = Value( size=len(encoded_value.data), - flags=set([Flag.LOST, Flag.STALE]), - int_flags={ - IntFlag.CLIENT_FLAG: encoded_value.encoding_id, - IntFlag.RETURNED_CAS_TOKEN: expected_cas_token, - }, + value=None, + win=False, + stale=True, + cas_token=expected_cas_token, + client_flag=encoded_value.encoding_id, ) memcache_socket.get_value.return_value = encoded_value.data @@ -765,10 +760,9 @@ def test_get_or_lease_hit( encoded_value = MixedSerializer().serialize(expected_value) memcache_socket.get_response.return_value = Value( size=len(encoded_value.data), - int_flags={ - IntFlag.CLIENT_FLAG: encoded_value.encoding_id, - IntFlag.RETURNED_CAS_TOKEN: expected_cas_token, - }, + value=None, + cas_token=expected_cas_token, + client_flag=encoded_value.encoding_id, ) memcache_socket.get_value.return_value = encoded_value.data @@ -790,10 +784,9 @@ def test_get_or_lease_miss_win( expected_cas_token = 123 memcache_socket.get_response.return_value = Value( size=0, - flags=set([Flag.WIN]), - int_flags={ - IntFlag.RETURNED_CAS_TOKEN: expected_cas_token, - }, + value=None, + win=True, + cas_token=expected_cas_token, ) memcache_socket.get_value.return_value = b"" @@ -818,24 +811,21 @@ def test_get_or_lease_miss_lost_then_data( memcache_socket.get_response.side_effect = [ Value( size=0, - flags=set([Flag.LOST]), - int_flags={ - IntFlag.RETURNED_CAS_TOKEN: expected_cas_token - 1, - }, + value=None, + win=False, + cas_token=expected_cas_token - 1, ), Value( size=0, - flags=set([Flag.LOST]), - int_flags={ - IntFlag.RETURNED_CAS_TOKEN: expected_cas_token - 1, - }, + value=None, + win=False, + cas_token=expected_cas_token - 1, ), Value( size=len(encoded_value.data), - int_flags={ - IntFlag.CLIENT_FLAG: encoded_value.encoding_id, - IntFlag.RETURNED_CAS_TOKEN: expected_cas_token, - }, + value=None, + cas_token=expected_cas_token, + client_flag=encoded_value.encoding_id, ), ] memcache_socket.get_value.side_effect = [b"", b"", encoded_value.data] @@ -871,24 +861,21 @@ def test_get_or_lease_miss_lost_then_win( memcache_socket.get_response.side_effect = [ Value( size=0, - flags=set([Flag.LOST]), - int_flags={ - IntFlag.RETURNED_CAS_TOKEN: expected_cas_token - 1, - }, + value=None, + win=False, + cas_token=expected_cas_token - 1, ), Value( size=0, - flags=set([Flag.LOST]), - int_flags={ - IntFlag.RETURNED_CAS_TOKEN: expected_cas_token - 1, - }, + value=None, + win=False, + cas_token=expected_cas_token - 1, ), Value( size=0, - flags=set([Flag.WIN]), - int_flags={ - IntFlag.RETURNED_CAS_TOKEN: expected_cas_token, - }, + value=None, + win=True, + cas_token=expected_cas_token, ), ] memcache_socket.get_value.side_effect = [b"", b"", b""] @@ -923,10 +910,9 @@ def test_get_or_lease_miss_runs_out_of_retries( expected_cas_token = 123 memcache_socket.get_response.return_value = Value( size=0, - flags=set([Flag.LOST]), - int_flags={ - IntFlag.RETURNED_CAS_TOKEN: expected_cas_token, - }, + value=None, + win=False, + cas_token=expected_cas_token, ) memcache_socket.get_value.return_value = b"" @@ -1223,7 +1209,7 @@ def test_delta_cmd(memcache_socket: MemcacheSocket, cache_client: CacheClient) - memcache_socket.sendall.reset_mock() memcache_socket.get_response.reset_mock() - memcache_socket.get_response.return_value = Value(size=2) + memcache_socket.get_response.return_value = Value(size=2, value=None) memcache_socket.get_value.return_value = b"10" result = cache_client.delta_and_get(key=Key("foo"), delta=1) @@ -1247,8 +1233,8 @@ def test_delta_cmd(memcache_socket: MemcacheSocket, cache_client: CacheClient) - def test_multi_get(memcache_socket: MemcacheSocket, cache_client: CacheClient) -> None: memcache_socket.get_response.side_effect = [ Miss(), - Value(size=2, int_flags={IntFlag.CLIENT_FLAG: MixedSerializer.BINARY}), - Value(size=2, flags=set([Flag.WIN])), + Value(size=2, value=None, client_flag=MixedSerializer.BINARY), + Value(size=2, value=None, win=True), ] memcache_socket.get_value.return_value = b"OK" diff --git a/tests/memcache_socket_test.py b/tests/memcache_socket_test.py index 202b3aa..796b21b 100644 --- a/tests/memcache_socket_test.py +++ b/tests/memcache_socket_test.py @@ -8,13 +8,10 @@ from meta_memcache.errors import MemcacheError from meta_memcache.protocol import ( Conflict, - Flag, - IntFlag, Miss, NotStored, ServerVersion, Success, - TokenFlag, Value, ) @@ -70,11 +67,11 @@ def test_get_response( ms = MemcacheSocket(fake_socket) result = ms.get_response() assert isinstance(result, Success) - assert result.int_flags == {IntFlag.RETURNED_CAS_TOKEN: 1} + assert result.cas_token == 1 result = ms.get_response() assert isinstance(result, Value) - assert result.int_flags == {IntFlag.RETURNED_CAS_TOKEN: 1} + assert result.cas_token == 1 assert result.size == 2 @@ -87,11 +84,11 @@ def test_get_response_1_6_6( ms = MemcacheSocket(fake_socket, version=ServerVersion.AWS_1_6_6) result = ms.get_response() assert isinstance(result, Success) - assert result.int_flags == {IntFlag.RETURNED_CAS_TOKEN: 1} + assert result.cas_token == 1 result = ms.get_response() assert isinstance(result, Value) - assert result.int_flags == {IntFlag.RETURNED_CAS_TOKEN: 1} + assert result.cas_token == 1 assert result.size == 2 @@ -124,7 +121,7 @@ def test_get_value( ms = MemcacheSocket(fake_socket) result = ms.get_response() assert isinstance(result, Value) - assert result.int_flags == {IntFlag.RETURNED_CAS_TOKEN: 1} + assert result.cas_token == 1 assert result.size == 2 ms.get_value(2) @@ -138,9 +135,9 @@ def test_get_value_large( ms = MemcacheSocket(fake_socket, buffer_size=100) result = ms.get_response() assert isinstance(result, Value) - assert result.int_flags == {IntFlag.RETURNED_CAS_TOKEN: 1} - assert result.flags == set([Flag.WIN]) - assert result.token_flags == {TokenFlag.OPAQUE: b"xxx"} + assert result.cas_token == 1 + assert result.win is True + assert result.opaque == b"xxx" assert result.size == 200 value = ms.get_value(result.size) assert len(value) == result.size @@ -225,7 +222,8 @@ def test_reset_buffer( assert len(value) == result.size assert value == b"1234567890" * 5 ms._reset_buffer() - assert ms._pos == len(data) // 2 + assert ms._pos == 0 + result = ms.get_response() value = ms.get_value(result.size) assert len(value) == result.size diff --git a/tests/migrating_cache_client_test.py b/tests/migrating_cache_client_test.py index 9131d0e..533a34c 100644 --- a/tests/migrating_cache_client_test.py +++ b/tests/migrating_cache_client_test.py @@ -74,17 +74,13 @@ def _set_cache_client_mock_get_return_values(client: Mock, ttl: int = 10) -> Non client.meta_get.return_value = Value( size=3, value="bar", - flags=set(), - int_flags={IntFlag.TTL: ttl}, - token_flags={}, + ttl=ttl, ) client.meta_multiget.return_value = { Key(key="foo", routing_key=None, is_unicode=False): Value( size=3, value="bar", - flags=set(), - int_flags={IntFlag.TTL: ttl}, - token_flags={}, + ttl=ttl, ) } diff --git a/tests/probabilistic_hot_cache_test.py b/tests/probabilistic_hot_cache_test.py index a87b0fb..728bdc5 100644 --- a/tests/probabilistic_hot_cache_test.py +++ b/tests/probabilistic_hot_cache_test.py @@ -29,10 +29,8 @@ def meta_get( return Value( size=1, value=1, - int_flags={ - IntFlag.HIT_AFTER_WRITE: 1, - IntFlag.LAST_READ_AGE: 1, - }, + fetched=1, + last_access=1, ) elif key.key.endswith("miss"): return Miss() @@ -40,10 +38,8 @@ def meta_get( return Value( size=1, value=1, - int_flags={ - IntFlag.HIT_AFTER_WRITE: 1, - IntFlag.LAST_READ_AGE: 9999, - }, + fetched=1, + last_access=9999, ) def meta_multiget( @@ -643,10 +639,8 @@ def test_stale_expires( client.meta_get.side_effect = lambda *args, **kwargs: Value( size=1, value=1, - int_flags={ - IntFlag.HIT_AFTER_WRITE: 1, - IntFlag.LAST_READ_AGE: 9999, - }, + fetched=1, + last_access=9999, ) # The item will no longer be in the hot cache