Skip to content

Commit

Permalink
Fix failing tests
Browse files Browse the repository at this point in the history
  • Loading branch information
Vasil Pashov committed Mar 28, 2024
1 parent 7b0895f commit 81871d9
Show file tree
Hide file tree
Showing 4 changed files with 17 additions and 24 deletions.
2 changes: 1 addition & 1 deletion cpp/arcticdb/python/normalization_checks.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ void fix_normalization_or_throw(
auto &old_norm = existing_isr.tsd().proto().normalization();
auto &new_norm = new_frame.norm_meta;
if (check_pandas_like(old_norm, new_norm)) {
const IndexDescriptor::Type old_index_type = existing_isr.seg().descriptor().index().type();
const IndexDescriptor::Type old_index_type = existing_isr.tsd().proto().stream_descriptor().index().kind();
const IndexDescriptor::Type new_index_type = new_frame.desc.index().type();
if (old_index_type == new_index_type && old_index_type == IndexDescriptor::ROWCOUNT) {
update_rowcount_normalization_data(old_norm, new_norm, existing_isr.tsd().proto().total_rows());
Expand Down
8 changes: 4 additions & 4 deletions cpp/arcticdb/stream/index.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -313,19 +313,19 @@ class EmptyIndex : public BaseIndex<EmptyIndex> {
}

[[nodiscard]] static IndexValue start_value_for_segment(const SegmentInMemory& segment) {
return static_cast<timestamp>(segment.offset());
return static_cast<NumericIndex>(segment.offset());
}

[[nodiscard]] static IndexValue end_value_for_segment(const SegmentInMemory& segment) {
return static_cast<timestamp>(segment.offset());
return static_cast<NumericIndex>(segment.offset());
}

[[nodiscard]] static IndexValue start_value_for_keys_segment(const SegmentInMemory& segment) {
return static_cast<timestamp>(segment.offset());
return static_cast<NumericIndex>(segment.offset());
}

[[nodiscard]] static IndexValue end_value_for_keys_segment(const SegmentInMemory& segment) {
return static_cast<timestamp>(segment.offset());
return static_cast<NumericIndex>(segment.offset());
}
};

Expand Down
23 changes: 11 additions & 12 deletions python/arcticdb/version_store/_normalization.py
Original file line number Diff line number Diff line change
Expand Up @@ -292,20 +292,19 @@ def _normalize_single_index(index, index_names, index_norm, dynamic_strings=None
# index: pd.Index or np.ndarray -> np.ndarray
index_tz = None

if not index_norm.is_physically_stored:
if isinstance(index_norm, NormalizationMetadata.PandasIndex) and not index_norm.is_physically_stored:
if index.name:
if not isinstance(index.name, int) and not isinstance(index.name, str):
raise NormalizationException(
f"Index name must be a string or an int, received {index.name} of type {type(index.name)}"
)
if isinstance(index.name, int):
index_norm.is_int = True
index_norm.name = str(index.name)
if isinstance(index, RangeIndex):
# skip index since we can reconstruct it, so no need to actually store it
if index.name:
if not isinstance(index.name, int) and not isinstance(index.name, str):
raise NormalizationException(
f"Index name must be a string or an int, received {index.name} of type {type(index.name)}"
)
if isinstance(index.name, int):
index_norm.is_int = True
index_norm.name = str(index.name)
index_norm.start = index.start if _range_index_props_are_public else index._start
index_norm.step = index.step if _range_index_props_are_public else index._step
return [], []
return [], []
else:
coerce_type = DTN64_DTYPE if len(index) == 0 else None
Expand Down Expand Up @@ -550,7 +549,7 @@ def _index_to_records(self, df, pd_norm, dynamic_strings, string_max_len):
is_not_range_index = not isinstance(index, RangeIndex)
df_has_rows = not(len(index) == 0 and len(df.select_dtypes(include="category").columns) == 0)
index_norm = pd_norm.index
index_norm.is_physically_stored = is_not_range_index or df_has_rows
index_norm.is_physically_stored = is_not_range_index and df_has_rows
if not df_has_rows:
index = Index([])

Expand Down Expand Up @@ -844,7 +843,6 @@ def normalize(self, item, string_max_len=None, dynamic_strings=False, coerce_col
# type: (DataFrame, Optional[int])->NormalizedInput
norm_meta = NormalizationMetadata()
norm_meta.df.common.mark = True

if isinstance(item.columns, RangeIndex):
norm_meta.df.has_synthetic_columns = True

Expand Down Expand Up @@ -1097,6 +1095,7 @@ def normalize(self, item, string_max_len=None, dynamic_strings=False, coerce_col
norm_meta = NormalizationMetadata()
norm_meta.ts.mark = True
index_norm = norm_meta.ts.common.index
index_norm.is_physically_stored = len(item.times) > 0 and not isinstance(item.times, RangeIndex)
index_names, ix_vals = _normalize_single_index(
item.times, ["times"], index_norm, dynamic_strings, string_max_len
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -816,10 +816,4 @@ def test_cannot_append_different_index_type_after_first_non_empty(self, lmdb_ver
lmdb_version_store_static_and_dynamic.write("sym", pd.DataFrame({"col": []}))
lmdb_version_store_static_and_dynamic.append("sym", pd.DataFrame({"col": [1,2,3]}, index=incompatible_indexes[0]))
with pytest.raises(Exception):
lmdb_version_store_static_and_dynamic.append("sym", pd.DataFrame({"col": [4, 5, 6]}, index=incompatible_indexes[1]))


def test_foo(lmdb_version_store_v2):
lmdb_version_store_v2.write("sym", pd.DataFrame({"col": []}, index=pd.DatetimeIndex([])))
print(lmdb_version_store_v2.read("sym").data)
print(lmdb_version_store_v2.read("sym").data.index)
lmdb_version_store_static_and_dynamic.append("sym", pd.DataFrame({"col": [4, 5, 6]}, index=incompatible_indexes[1]))

0 comments on commit 81871d9

Please sign in to comment.