Finish getitem for RawArray. (#16)

At the same level as the rest: no `newaxis` or `Ellipsis`. * Start working on RawArray. * Creating and testing C++ awkward arrays in PR016. * Some more tests before embarking on Slice. * Non-trivial test of RawArray. * A test that won't automatically fail. * Nearly all cases are covered. * Done with RawArray (skipping Ellipsis and newaxis, like the others).
scikit-hep · Oct 10, 2019 · 536cdb6 · 536cdb6
1 parent a4043b6
commit 536cdb6
Show file tree

Hide file tree

Showing 7 changed files with 233 additions and 40 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -43,6 +43,7 @@ add_library(awkward-static STATIC $<TARGET_OBJECTS:awkward-objects>)
 add_library(awkward        SHARED $<TARGET_OBJECTS:awkward-objects>)
 target_link_libraries(awkward-static PRIVATE awkward-cpu-kernels-static)
 target_link_libraries(awkward        PRIVATE awkward-cpu-kernels-static)
+addtest(PR016 tests/test_PR016_finish_getitem_for_rawarray.cpp)
 
 pybind11_add_module(layout src/pyawkward.cpp)
 set_target_properties(layout PROPERTIES CXX_VISIBILITY_PRESET default)

diff --git a/README.md b/README.md
@@ -63,7 +63,7 @@ Completed items are ☑check-marked. See [closed PRs](https://github.com/scikit-
    * [ ] Explicit broadcasting functions for jagged and non-jagged arrays and scalars.
    * [ ] Extend `__getitem__` to take jagged arrays of integers and booleans (same behavior as old).
    * [ ] Full suite of array types:
-      * [ ] `RawArray`: flat, 1-dimensional array type for pure C++ (header-only).
+      * [X] `RawArray`: flat, 1-dimensional array type for pure C++ (header-only).
       * [X] `NumpyArray`: rectilinear, N-dimensional array type without Python/pybind11 dependencies, but intended for Numpy.
       * [X] `ListArray`: the new `JaggedArray`, based on `starts` and `stops` (i.e. fully general).
       * [X] `ListOffsetArray`: the `JaggedArray` case with no unreachable data between reachable data (gaps).

diff --git a/VERSION_INFO b/VERSION_INFO
@@ -1 +1 @@
-0.1.12
+0.1.13
diff --git a/include/awkward/RawArray.h b/include/awkward/RawArray.h
@@ -47,29 +47,20 @@ namespace awkward {
         , itemsize_(sizeof(T)) { }
 
     const std::shared_ptr<T> ptr() const { return ptr_; }
-
     const int64_t offset() const { return offset_; }
-
     const int64_t itemsize() const { return itemsize_; }
 
     bool isempty() const { return length_ == 0; }
-
     ssize_t byteoffset() const { return (ssize_t)itemsize_*(ssize_t)offset_; }
-
     uint8_t* byteptr() const { return reinterpret_cast<uint8_t*>(reinterpret_cast<ssize_t>(ptr_.get()) + byteoffset()); }
-
     ssize_t bytelength() const { return (ssize_t)itemsize_*(ssize_t)length_; }
-
     uint8_t getbyte(ssize_t at) const { return *reinterpret_cast<uint8_t*>(reinterpret_cast<ssize_t>(ptr_.get()) + (ssize_t)(byteoffset() + at)); }
 
-    T* borrow() const { return borrow(0); }
-
     T* borrow(int64_t at) const { return reinterpret_cast<T*>(reinterpret_cast<ssize_t>(ptr_.get()) + (ssize_t)itemsize_*(ssize_t)(offset_ + at)); }
 
     virtual const std::string classname() const { return std::string("RawArrayOf<") + std::string(typeid(T).name()) + std::string(">"); }
 
     virtual const std::shared_ptr<Identity> id() const { return id_; }
-
     virtual void setid() {
       if (length() <= kMaxInt32) {
         Identity32* rawid = new Identity32(Identity::newref(), Identity::FieldLoc(), 1, length());
@@ -84,14 +75,14 @@ namespace awkward {
         setid(newid);
       }
     }
-
     virtual void setid(const std::shared_ptr<Identity> id) {
       if (id.get() != nullptr  &&  length() != id.get()->length()) {
         throw std::invalid_argument("content and its id must have the same length");
       }
       id_ = id;
     }
 
+    const std::string tostring() { return tostring_part("", "", ""); }
     virtual const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const {
       std::stringstream out;
       out << indent << pre << "<RawArray of=\"" << typeid(T).name() << "\" length=\"" << length_ << "\" itemsize=\"" << itemsize_ << "\" data=\"";
@@ -150,7 +141,7 @@ namespace awkward {
       if (!(0 <= regular_at  &&  regular_at < length_)) {
         util::handle_error(failure("index out of range", kSliceNone, at), classname(), id_.get());
       }
-      return getitem_range_unsafe(regular_at, regular_at + 1);
+      return getitem_at_unsafe(regular_at);
     }
 
     virtual const std::shared_ptr<Content> getitem_at_unsafe(int64_t at) const {
@@ -160,26 +151,26 @@ namespace awkward {
     virtual const std::shared_ptr<Content> getitem_range(int64_t start, int64_t stop) const {
       int64_t regular_start = start;
       int64_t regular_stop = stop;
-      awkward_regularize_rangeslice(regular_start, regular_stop, true, start != Slice::none(), stop != Slice::none(), length_);
+      awkward_regularize_rangeslice(&regular_start, &regular_stop, true, start != Slice::none(), stop != Slice::none(), length_);
+      if (id_.get() != nullptr  &&  regular_stop > id_.get()->length()) {
+        util::handle_error(failure("index out of range", kSliceNone, stop), id_.get()->classname(), nullptr);
+      }
       return getitem_range_unsafe(regular_start, regular_stop);
     }
 
     virtual const std::shared_ptr<Content> getitem_range_unsafe(int64_t start, int64_t stop) const {
       std::shared_ptr<Identity> id(nullptr);
       if (id_.get() != nullptr) {
-        if (regular_stop > id_.get()->length()) {
-          util::handle_error(failure("index out of range", kSliceNone, stop), id_.get()->classname(), nullptr);
-        }
-        id = id_.get()->getitem_range(regular_start, regular_stop);
+        id = id_.get()->getitem_range_unsafe(start, stop);
       }
-      return std::shared_ptr<Content>(new RawArrayOf<T>(id, ptr_, offset_ + regular_start, regular_stop - regular_start, itemsize_));
+      return std::shared_ptr<Content>(new RawArrayOf<T>(id, ptr_, offset_ + start, stop - start, itemsize_));
     }
 
     virtual const std::shared_ptr<Content> getitem(const Slice& where) const {
       std::shared_ptr<SliceItem> nexthead = where.head();
       Slice nexttail = where.tail();
       Index64 nextadvanced(0);
-      return getitem_next(nexthead, nexttail, nextadvanced, false);
+      return getitem_next(nexthead, nexttail, nextadvanced);
     }
 
     const std::shared_ptr<Content> getitem_next(const std::shared_ptr<SliceItem> head, const Slice& tail, const Index64& advanced) const {
@@ -188,7 +179,7 @@ namespace awkward {
       }
 
       if (head.get() == nullptr) {
-        throw std::runtime_error("null");
+        return shallow_copy();
       }
 
       else if (SliceAt* at = dynamic_cast<SliceAt*>(head.get())) {
@@ -209,23 +200,21 @@ namespace awkward {
           else if (step == 0) {
             throw std::invalid_argument("slice step must not be 0");
           }
-          awkward_regularize_rangeslice(start, stop, step > 0, start != Slice::none(), stop != Slice::none(), length_);
-
-          throw std::runtime_error("stop here for now");
-
-
+          awkward_regularize_rangeslice(&start, &stop, step > 0, range->hasstart(), range->hasstop(), length_);
+
+          int64_t numer = abs(start - stop);
+          int64_t denom = abs(step);
+          int64_t d = numer / denom;
+          int64_t m = numer % denom;
+          int64_t lenhead = d + (m != 0 ? 1 : 0);
+
+          Index64 nextcarry(lenhead);
+          int64_t* nextcarryptr = nextcarry.ptr().get();
+          for (int64_t i = 0;  i < lenhead;  i++) {
+            nextcarryptr[i] = start + step*i;
+          }
 
-          // int64_t regular_start = start;
-          // int64_t regular_stop = stop;
-          // awkward_regularize_rangeslice(regular_start, regular_stop, true, start != Slice::none(), stop != Slice::none(), length_);
-          // std::shared_ptr<Identity> id(nullptr);
-          // if (id_.get() != nullptr) {
-          //   if (regular_stop > id_.get()->length()) {
-          //     throw std::invalid_argument("index out of range for identity");
-          //   }
-          //   id = id_.get()->slice(regular_start, regular_stop);
-          // }
-          // return std::shared_ptr<Content>(new RawArrayOf<T>(id, ptr_, offset_ + regular_start, regular_stop - regular_start, itemsize_));
+          return carry(nextcarry);
         }
       }
 
@@ -238,7 +227,17 @@ namespace awkward {
       }
 
       else if (SliceArray64* array = dynamic_cast<SliceArray64*>(head.get())) {
-        throw std::runtime_error("array");
+        assert(advanced.length() == 0);
+        if (array->shape().size() != 1) {
+          throw std::runtime_error("array.ndim != 1");
+        }
+        Index64 flathead = array->ravel();
+        Error err = awkward_regularize_arrayslice_64(
+          flathead.ptr().get(),
+          flathead.length(),
+          length_);
+        util::handle_error(err, classname(), id_.get());
+        return carry(flathead);
       }
 
       else {
@@ -247,7 +246,22 @@ namespace awkward {
     }
 
     virtual const std::shared_ptr<Content> carry(const Index64& carry) const {
-      throw std::runtime_error("RawArray<T>::carry");
+      std::shared_ptr<T> ptr(new T[(size_t)carry.length()], awkward::util::array_deleter<T>());
+      Error err = awkward_numpyarray_getitem_next_null_64(
+        reinterpret_cast<uint8_t*>(ptr.get()),
+        reinterpret_cast<uint8_t*>(ptr_.get()),
+        carry.length(),
+        itemsize_,
+        byteoffset(),
+        carry.ptr().get());
+      util::handle_error(err, classname(), id_.get());
+
+      std::shared_ptr<Identity> id(nullptr);
+      if (id_.get() != nullptr) {
+        id = id_.get()->getitem_carry_64(carry);
+      }
+
+      return std::shared_ptr<Content>(new RawArrayOf<T>(id, ptr, 0, carry.length(), itemsize_));
     }
 
     virtual const std::pair<int64_t, int64_t> minmax_depth() const { return std::pair<int64_t, int64_t>(1, 1); }

diff --git a/include/awkward/Slice.h b/include/awkward/Slice.h
@@ -16,13 +16,17 @@ namespace awkward {
   public:
     static int64_t none() { return kSliceNone; }
 
+    virtual const std::shared_ptr<SliceItem> shallow_copy() const = 0;
     virtual const std::string tostring() const = 0;
   };
 
   class SliceAt: public SliceItem {
   public:
     SliceAt(int64_t at): at_(at) { }
     int64_t at() const { return at_; }
+    virtual const std::shared_ptr<SliceItem> shallow_copy() const {
+      return std::shared_ptr<SliceItem>(new SliceAt(at_));
+    }
     virtual const std::string tostring() const;
   private:
     const int64_t at_;
@@ -38,6 +42,9 @@ namespace awkward {
     int64_t step() const { return step_; }
     bool hasstart() const { return start_ != none(); }
     bool hasstop() const { return stop_ != none(); }
+    virtual const std::shared_ptr<SliceItem> shallow_copy() const {
+      return std::shared_ptr<SliceItem>(new SliceRange(start_, stop_, step_));
+    }
     virtual const std::string tostring() const;
   private:
     const int64_t start_;
@@ -48,12 +55,18 @@ namespace awkward {
   class SliceEllipsis: public SliceItem {
   public:
     SliceEllipsis() { }
+    virtual const std::shared_ptr<SliceItem> shallow_copy() const {
+      return std::shared_ptr<SliceItem>(new SliceEllipsis());
+    }
     virtual const std::string tostring() const;
   };
 
   class SliceNewAxis: public SliceItem {
   public:
     SliceNewAxis() { }
+    virtual const std::shared_ptr<SliceItem> shallow_copy() const {
+      return std::shared_ptr<SliceItem>(new SliceNewAxis());
+    }
     virtual const std::string tostring() const;
   };
 
@@ -69,6 +82,9 @@ namespace awkward {
     const std::vector<int64_t> shape() const { return shape_; }
     const std::vector<int64_t> strides() const { return strides_; }
     int64_t ndim() const { return (int64_t)shape_.size(); }
+    virtual const std::shared_ptr<SliceItem> shallow_copy() const {
+      return std::shared_ptr<SliceItem>(new SliceArrayOf<T>(index_, shape_, strides_));
+    }
     virtual const std::string tostring() const;
     const std::string tostring_part() const;
     const IndexOf<T> ravel() const;

diff --git a/src/libawkward/Index.cpp b/src/libawkward/Index.cpp
@@ -90,7 +90,7 @@ namespace awkward {
 
   template <typename T>
   IndexOf<T> IndexOf<T>::getitem_range_unsafe(int64_t start, int64_t stop) const {
-    assert(0 <= start  &&  start < length_  &&  0 <= stop  &&  stop < length_);
+    assert(0 <= start  &&  start < length_  &&  start <= stop  &&  stop <= length_);
     return IndexOf<T>(ptr_, offset_ + start*(start != stop), stop - start);
   }