[io] add test bed for streaming large objects

cmake/modules/RootCTest.cmake

@@ -51,17 +51,19 @@ foreach(d ${test_list})
   endif()
 endforeach()
 
-# When ninja is in use, tests that compile an executable might try to rebuild the entire build tree.
-# If multiple of these are invoked in parallel, ninja will suffer from race conditions.
+# When ninja or the Microsoft generator are in use, tests that compile an executable might try
+# to rebuild the entire build tree. If multiple of these are invoked in parallel, ninja will
+# suffer from race conditions.
 # To solve this, do the following:
 # - Add a test that updates the build tree (equivalent to "ninja all"). This one will run in complete isolation.
 # - Make all tests that require a ninja build depend on the above test.
 # - Use a RESOURCE_LOCK on all tests that invoke ninja, so no two tests will invoke ninja in parallel
-if(CMAKE_GENERATOR MATCHES Ninja)
-  add_test(NAME ninja-build-all
-      COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR})
-  set_tests_properties(ninja-build-all PROPERTIES
-      RESOURCE_LOCK NINJA_BUILD
-      FIXTURES_SETUP NINJA_BUILD_ALL
+if(GeneratorNeedsBuildSerialization)
+  add_test(NAME cmake-build-all
+      COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --config ${build_configuration})
+  set_tests_properties(cmake-build-all PROPERTIES
+      RESOURCE_LOCK CMAKE_BUILD
+      FIXTURES_SETUP CMAKE_BUILD_ALL
       RUN_SERIAL True)
+  set(GeneratorNeedsBuildSerialization True)
 endif()

cmake/modules/RootMacros.cmake

@@ -45,6 +45,10 @@ else()
   set(ROOT_LIBRARY_PROPERTIES ${ROOT_LIBRARY_PROPERTIES} ${ROOT_LIBRARY_PROPERTIES_NO_VERSION} )
 endif()
 
+if (NOT DEFINED ROOT_root_CMD)
+  set(ROOT_root_CMD $<TARGET_FILE:root.exe>)
+endif()
+
 include(CMakeParseArguments)
 
 #---------------------------------------------------------------------------------------------------
@@ -2657,9 +2661,9 @@ macro(ROOTTEST_GENERATE_DICTIONARY dictname)
                                     -- ${always-make})
 
   set_property(TEST ${GENERATE_DICTIONARY_TEST} PROPERTY ENVIRONMENT ${ROOTTEST_ENVIRONMENT})
-  if(CMAKE_GENERATOR MATCHES Ninja)
-    set_property(TEST ${GENERATE_DICTIONARY_TEST} APPEND PROPERTY RESOURCE_LOCK NINJA_BUILD)
-    set_property(TEST ${GENERATE_DICTIONARY_TEST} APPEND PROPERTY FIXTURES_REQUIRED NINJA_BUILD_ALL)
+  if(GeneratorNeedsBuildSerialization)
+    set_property(TEST ${GENERATE_DICTIONARY_TEST} APPEND PROPERTY RESOURCE_LOCK CMAKE_BUILD)
+    set_property(TEST ${GENERATE_DICTIONARY_TEST} APPEND PROPERTY FIXTURES_REQUIRED CMAKE_BUILD_ALL)
   endif()
 
   if (ARG_FIXTURES_SETUP)
@@ -2765,9 +2769,9 @@ macro(ROOTTEST_GENERATE_REFLEX_DICTIONARY dictionary)
                                     -- ${always-make})
 
   set_property(TEST ${GENERATE_REFLEX_TEST} PROPERTY ENVIRONMENT ${ROOTTEST_ENVIRONMENT})
-  if(CMAKE_GENERATOR MATCHES Ninja)
-    set_property(TEST ${GENERATE_REFLEX_TEST} APPEND PROPERTY RESOURCE_LOCK NINJA_BUILD)
-    set_property(TEST ${GENERATE_REFLEX_TEST} APPEND PROPERTY FIXTURES_REQUIRED NINJA_BUILD_ALL)
+  if(GeneratorNeedsBuildSerialization)
+    set_property(TEST ${GENERATE_REFLEX_TEST} APPEND PROPERTY RESOURCE_LOCK CMAKE_BUILD)
+    set_property(TEST ${GENERATE_REFLEX_TEST} APPEND PROPERTY FIXTURES_REQUIRED CMAKE_BUILD_ALL)
   endif()
 
   if (ARG_FIXTURES_SETUP)
@@ -2877,9 +2881,9 @@ macro(ROOTTEST_GENERATE_EXECUTABLE executable)
       RESOURCE_LOCK ${ARG_RESOURCE_LOCK})
   endif()
 
-  if(CMAKE_GENERATOR MATCHES Ninja)
-    set_property(TEST ${GENERATE_EXECUTABLE_TEST} APPEND PROPERTY RESOURCE_LOCK NINJA_BUILD)
-    set_property(TEST ${GENERATE_EXECUTABLE_TEST} APPEND PROPERTY FIXTURES_REQUIRED NINJA_BUILD_ALL)
+  if(GeneratorNeedsBuildSerialization)
+    set_property(TEST ${GENERATE_EXECUTABLE_TEST} APPEND PROPERTY RESOURCE_LOCK CMAKE_BUILD)
+    set_property(TEST ${GENERATE_EXECUTABLE_TEST} APPEND PROPERTY FIXTURES_REQUIRED CMAKE_BUILD_ALL)
   endif()
 
   if(MSVC AND NOT CMAKE_GENERATOR MATCHES Ninja)

core/base/inc/TBuffer.h

@@ -64,11 +64,20 @@ class TBuffer : public TObject {
    void operator=(const TBuffer &) = delete;
 
    Int_t Read(const char *name) override { return TObject::Read(name); }
-   Int_t Write(const char *name, Int_t opt, Int_t bufsize) override
+   Int_t Write(const char *name, Int_t opt, Long64_t bufsize) override
                               { return TObject::Write(name, opt, bufsize); }
-   Int_t Write(const char *name, Int_t opt, Int_t bufsize) const override
+   Int_t Write(const char *name, Int_t opt, Long64_t bufsize) const override
                               { return TObject::Write(name, opt, bufsize); }
 
+   ////////////////////////////////////////////////////////////////////////////////
+   /// Reserve space for a leading byte count and return the position where to
+   /// store the byte count value.
+   ///
+   /// \param[in] cl The class for which we are reserving the byte count, used for error reporting.
+   /// \return The position (cntpos) where the byte count should be stored later,
+   ///         or kOverflowPosition if the position exceeds kMaxCountPosition
+   virtual UInt_t ReserveByteCount(const TClass *) = 0;
+
 public:
    enum EMode { kRead = 0, kWrite = 1 };
    enum EStatusBits {
@@ -78,53 +87,105 @@ class TBuffer : public TObject {
    enum { kInitialSize = 1024, kMinimalSize = 128 };
 
    TBuffer(EMode mode);
-   TBuffer(EMode mode, Int_t bufsize);
-   TBuffer(EMode mode, Int_t bufsize, void *buf, Bool_t adopt = kTRUE, ReAllocCharFun_t reallocfunc = nullptr);
+   TBuffer(EMode mode, Long64_t bufsize);
+   TBuffer(EMode mode, Long64_t bufsize, void *buf, Bool_t adopt = kTRUE, ReAllocCharFun_t reallocfunc = nullptr);
    virtual ~TBuffer();
 
    Int_t    GetBufferVersion() const { return fVersion; }
    Bool_t   IsReading() const { return (fMode & kWrite) == 0; }
    Bool_t   IsWriting() const { return (fMode & kWrite) != 0; }
    void     SetReadMode();
    void     SetWriteMode();
-   void     SetBuffer(void *buf, UInt_t bufsize = 0, Bool_t adopt = kTRUE, ReAllocCharFun_t reallocfunc = nullptr);
+   void     SetBuffer(void *buf, Long64_t bufsize = 0, Bool_t adopt = kTRUE, ReAllocCharFun_t reallocfunc = nullptr);
    ReAllocCharFun_t GetReAllocFunc() const;
    void     SetReAllocFunc(ReAllocCharFun_t reallocfunc = nullptr);
-   void     SetBufferOffset(Int_t offset = 0) { fBufCur = fBuffer+offset; }
+   void     SetBufferOffset(Long64_t offset = 0) { fBufCur = fBuffer+offset; }
    void     SetParent(TObject *parent);
    TObject *GetParent()  const;
    char    *Buffer()     const { return fBuffer; }
    char    *GetCurrent() const { return fBufCur; }
    Int_t    BufferSize() const { return fBufSize; }
    void     DetachBuffer() { fBuffer = nullptr; }
    Int_t    Length()     const { return (Int_t)(fBufCur - fBuffer); }
-   void     Expand(Int_t newsize, Bool_t copy = kTRUE);  // expand buffer to newsize
-   void     AutoExpand(Int_t size_needed);  // expand buffer to newsize
+   void     Expand(Long64_t newsize, Bool_t copy = kTRUE);  // expand buffer to newsize
+   void     AutoExpand(Long64_t size_needed);  // expand buffer to newsize
    Bool_t   ByteSwapBuffer(Long64_t n, EDataType type);  // Byte-swap N primitive-elements in the buffer
 
    virtual Bool_t     CheckObject(const TObject *obj) = 0;
    virtual Bool_t     CheckObject(const void *obj, const TClass *ptrClass) = 0;
 
-   virtual Int_t      ReadBuf(void *buf, Int_t max) = 0;
-   virtual void       WriteBuf(const void *buf, Int_t max) = 0;
+   virtual Long64_t   ReadBuf(void *buf, Long64_t max) = 0;
+   virtual void       WriteBuf(const void *buf, Long64_t max) = 0;
 
-   virtual char      *ReadString(char *s, Int_t max) = 0;
+   virtual char      *ReadString(char *s, Long64_t max) = 0;
    virtual void       WriteString(const char *s) = 0;
 
    virtual Int_t      GetVersionOwner() const  = 0;
    virtual Int_t      GetMapCount() const  = 0;
    virtual void       GetMappedObject(UInt_t tag, void* &ptr, TClass* &ClassPtr) const = 0;
-   virtual void       MapObject(const TObject *obj, UInt_t offset = 1) = 0;
-   virtual void       MapObject(const void *obj, const TClass *cl, UInt_t offset = 1) = 0;
+   virtual void       MapObject(const TObject *obj, ULong64_t offset = 1) = 0;
+   virtual void       MapObject(const void *obj, const TClass *cl, ULong64_t offset = 1) = 0;
    virtual void       Reset() = 0;
    virtual void       InitMap() = 0;
    virtual void       ResetMap() = 0;
    virtual void       SetReadParam(Int_t mapsize) = 0;
    virtual void       SetWriteParam(Int_t mapsize) = 0;
 
-   virtual Int_t      CheckByteCount(UInt_t startpos, UInt_t bcnt, const TClass *clss) = 0;
-   virtual Int_t      CheckByteCount(UInt_t startpos, UInt_t bcnt, const char *classname) = 0;
-   virtual void       SetByteCount(UInt_t cntpos, Bool_t packInVersion = kFALSE)= 0;
+   virtual Long64_t   CheckByteCount(ULong64_t startpos, ULong64_t bcnt, const TClass *clss) = 0;
+   virtual Long64_t   CheckByteCount(ULong64_t startpos, ULong64_t bcnt, const char *classname) = 0;
+   virtual void       SetByteCount(ULong64_t cntpos, Bool_t packInVersion = kFALSE)= 0;
+
+   /// \class TBuffer::ByteCountWriter
+   /// \ingroup Base
+   /// \brief RAII helper to automatically write the byte count for an object
+   /// to be used in the rare case where writing the class version number
+   /// and the byte count are decoupled.
+   ///
+   /// `ByteCountWriter` encapsulates the  pattern:
+   /// 1. Reserve space for a leading byte count with ReserveByteCount().
+   /// 2. Stream the object content.
+   /// 3. Finalize the byte count with SetByteCount().
+   ///
+   /// \note Create the instance as a local variable and keep it alive until all
+   ///       the bytes that should be counted have been written to the buffer.
+   ///
+   /// ### Example
+   /// \code{.cpp}
+   /// void MyClass::Streamer(TBuffer &b)
+   /// {
+   ///    if (b.IsWriting()) {
+   ///       // Reserve space for the byte count and auto-finalize on scope exit.
+   ///       TBuffer::ByteCountWriter bcnt(b, MyClass::Class());
+   ///
+   ///       b.WriteClass(MyClass::Class());
+   ///       // ... stream members ...
+   ///    } else {
+   ///       // ... read members ...
+   ///    }
+   /// }
+   /// \endcode
+   class ByteCountWriter {
+      TBuffer      &fBuffer;
+      Bool_t        fPackInVersion;
+      ULong64_t     fCntPos;
+   public:
+      ByteCountWriter() = delete;
+      ByteCountWriter(const ByteCountWriter&) = delete;
+      ByteCountWriter& operator=(const ByteCountWriter&) = delete;
+      ByteCountWriter(ByteCountWriter&&) = delete;
+      ByteCountWriter& operator=(ByteCountWriter&&) = delete;
+
+      ByteCountWriter(TBuffer &buf, const TClass *cl, Bool_t packInVersion = kFALSE) : fBuffer(buf), fPackInVersion(packInVersion) {
+         // We could split ReserveByteCount in a 32bit version that uses
+         // the ByteCountStack and another version that always returns the
+         // long range position.  For now keep it 'simpler' by always using
+         // the stack.
+         fCntPos = fBuffer.ReserveByteCount(cl);
+      }
+      ~ByteCountWriter() {
+         fBuffer.SetByteCount(fCntPos, fPackInVersion);
+      }
+   };
 
    virtual void       SkipVersion(const TClass *cl = nullptr) = 0;
    virtual Version_t  ReadVersion(UInt_t *start = nullptr, UInt_t *bcnt = nullptr, const TClass *cl = nullptr) = 0;
@@ -135,6 +196,23 @@ class TBuffer : public TObject {
 
    virtual void      *ReadObjectAny(const TClass* cast) = 0;
    virtual void       SkipObjectAny() = 0;
+   /** \brief Skip, based on a known start position and byte count, to the end of the current object.
+    *
+    * \warning Advanced use only.
+    *
+    * This overload exists primarily for error handling within a Streamer.
+    *
+    * A typical use case is a Streamer with a flow like:
+    *   - Read version and bytecount
+    *   - Start reading the data
+    *   - Detect an error (e.g. missing some information for a CollectionProxy)
+    *   - Properly set the cursor to the end of the object to allow reading to continue.
+    *
+    * Because the actual byte count information for \e large byte counts is kept only on the
+    * internal byte-count stack, there are only two viable options to support this use case:
+    * provide this overload, or make the stack accessible (e.g. via a getter).
+    */
+   virtual void       SkipObjectAny(Long64_t start, UInt_t bytecount) = 0;
 
    virtual void       TagStreamerInfo(TVirtualStreamerInfo* info) = 0;
    virtual void       IncrementLevel(TVirtualStreamerInfo* info) = 0;
@@ -164,7 +242,7 @@ class TBuffer : public TObject {
    virtual void       SetPidOffset(UShort_t offset) = 0;
    virtual Int_t      GetBufferDisplacement() const  = 0;
    virtual void       SetBufferDisplacement() = 0;
-   virtual void       SetBufferDisplacement(Int_t skipped) = 0;
+   virtual void       SetBufferDisplacement(Long64_t skipped) = 0;
 
    // basic types and arrays of basic types
    virtual   void     ReadFloat16 (Float_t *f, TStreamerElement *ele = nullptr) = 0;
@@ -320,8 +398,8 @@ class TBuffer : public TObject {
    // Utilities for TStreamerInfo
    virtual   void     ForceWriteInfo(TVirtualStreamerInfo *info, Bool_t force) = 0;
    virtual   void     ForceWriteInfoClones(TClonesArray *a) = 0;
-   virtual   Int_t    ReadClones (TClonesArray *a, Int_t nobjects, Version_t objvers) = 0;
-   virtual   Int_t    WriteClones(TClonesArray *a, Int_t nobjects) = 0;
+   virtual   Int_t    ReadClones (TClonesArray *a, Long64_t nobjects, Version_t objvers) = 0;
+   virtual   Int_t    WriteClones(TClonesArray *a, Long64_t nobjects) = 0;
 
    // Utilities for TClass
    virtual   Int_t    ReadClassEmulated(const TClass *cl, void *object, const TClass *onfile_class = nullptr) = 0;

core/base/inc/TDirectory.h

@@ -254,20 +254,20 @@ can be replaced with the simpler and exception safe:
    virtual void        Save() {}
    virtual Int_t       SaveObjectAs(const TObject * /*obj*/, const char * /*filename*/="", Option_t * /*option*/="") const;
    virtual void        SaveSelf(Bool_t /*force*/ = kFALSE) {}
-   virtual void        SetBufferSize(Int_t /* bufsize */) {}
+   virtual void        SetBufferSize(Long64_t /* bufsize */) {}
    virtual void        SetModified() {}
    virtual void        SetMother(TObject *mother) {fMother = (TObject*)mother;}
            void        SetName(const char* newname) override;
    virtual void        SetTRefAction(TObject * /*ref*/, TObject * /*parent*/) {}
    virtual void        SetSeekDir(Long64_t) {}
    virtual void        SetWritable(Bool_t) {}
            Int_t       Sizeof() const override {return 0;}
-   virtual Int_t       Write(const char * /*name*/=nullptr, Int_t /*opt*/=0, Int_t /*bufsize*/=0) override {return 0;}
-   virtual Int_t       Write(const char * /*name*/=nullptr, Int_t /*opt*/=0, Int_t /*bufsize*/=0) const override {return 0;}
-   virtual Int_t       WriteTObject(const TObject *obj, const char *name =nullptr, Option_t * /*option*/="", Int_t /*bufsize*/ =0);
+   virtual Int_t       Write(const char * /*name*/=nullptr, Int_t /*opt*/=0, Long64_t /*bufsize*/=0) override {return 0;}
+   virtual Int_t       Write(const char * /*name*/=nullptr, Int_t /*opt*/=0, Long64_t /*bufsize*/=0) const override {return 0;}
+   virtual Int_t       WriteTObject(const TObject *obj, const char *name =nullptr, Option_t * /*option*/="", Long64_t /*bufsize*/ =0);
 private:
 /// \cond HIDDEN_SYMBOLS
-           Int_t       WriteObject(void *obj, const char* name, Option_t *option="", Int_t bufsize=0); // Intentionally not implemented.
+           Int_t       WriteObject(void *obj, const char* name, Option_t *option="", Long64_t bufsize=0); // Intentionally not implemented.
 /// \endcond
 public:
    /// \brief Write an object with proper type checking.
@@ -280,7 +280,7 @@ can be replaced with the simpler and exception safe:
    /// from TObject. The method redirects to TDirectory::WriteObjectAny.
    template <typename T>
    inline std::enable_if_t<!std::is_base_of<TObject, T>::value, Int_t>
-   WriteObject(const T *obj, const char *name, Option_t *option = "", Int_t bufsize = 0)
+   WriteObject(const T *obj, const char *name, Option_t *option = "", Long64_t bufsize = 0)
    {
       return WriteObjectAny(obj, TClass::GetClass<T>(), name, option, bufsize);
    }
@@ -294,12 +294,12 @@ can be replaced with the simpler and exception safe:
    /// TObject. The method redirects to TDirectory::WriteTObject.
    template <typename T>
    inline std::enable_if_t<std::is_base_of<TObject, T>::value, Int_t>
-   WriteObject(const T *obj, const char *name, Option_t *option = "", Int_t bufsize = 0)
+   WriteObject(const T *obj, const char *name, Option_t *option = "", Long64_t bufsize = 0)
    {
       return WriteTObject(obj, name, option, bufsize);
    }
-   virtual Int_t       WriteObjectAny(const void *, const char * /*classname*/, const char * /*name*/, Option_t * /*option*/="", Int_t /*bufsize*/ =0) {return 0;}
-   virtual Int_t       WriteObjectAny(const void *, const TClass * /*cl*/, const char * /*name*/, Option_t * /*option*/="", Int_t /*bufsize*/ =0) {return 0;}
+   virtual Int_t       WriteObjectAny(const void *, const char * /*classname*/, const char * /*name*/, Option_t * /*option*/="", Long64_t /*bufsize*/ =0) {return 0;}
+   virtual Int_t       WriteObjectAny(const void *, const TClass * /*cl*/, const char * /*name*/, Option_t * /*option*/="", Long64_t /*bufsize*/ =0) {return 0;}
    virtual void        WriteDirHeader() {}
    virtual void        WriteKeys() {}
 

core/base/inc/TObject.h

@@ -172,8 +172,8 @@ class TObject {
    virtual void        SetDrawOption(Option_t *option="");  // *MENU*
    virtual void        SetUniqueID(UInt_t uid);
    virtual void        UseCurrentStyle();
-   virtual Int_t       Write(const char *name = nullptr, Int_t option = 0, Int_t bufsize = 0);
-   virtual Int_t       Write(const char *name = nullptr, Int_t option = 0, Int_t bufsize = 0) const;
+   virtual Int_t       Write(const char *name = nullptr, Int_t option = 0, Long64_t bufsize = 0);
+   virtual Int_t       Write(const char *name = nullptr, Int_t option = 0, Long64_t bufsize = 0) const;
 
    /// IsDestructed
    ///