diff --git a/.gitignore b/.gitignore index f0494a34..7dd3dc60 100644 --- a/.gitignore +++ b/.gitignore @@ -2,11 +2,7 @@ .git/** **/__pycache__/** **.coverage -.coverage -multipy/runtime/interpreter/cpython -multipy/runtime/interpreter/cpython/** **/build/** **/CMakeFiles/** -multipy/runtime/interpreter/frozen/** multipy/runtime/example/generated/ *.egg-info diff --git a/README.md b/README.md index 499d554a..97dc3633 100644 --- a/README.md +++ b/README.md @@ -153,7 +153,7 @@ cmake --build . --config Release ### Running unit tests for `multipy::runtime` -We first need to generate the neccessary examples. First make sure your python enviroment has [torch](https://pytorch.org). Afterwards, once `multipy::runtime` is built, run the following (executed automatically for `docker` and `pip` above): +We first need to generate the neccessary examples. First make sure your python environment has [torch](https://pytorch.org). Afterwards, once `multipy::runtime` is built, run the following (executed automatically for `docker` and `pip` above): ``` cd multipy/multipy/runtime diff --git a/multipy/runtime/deploy.h b/multipy/runtime/deploy.h index 0e5ff225..81a5b3fb 100644 --- a/multipy/runtime/deploy.h +++ b/multipy/runtime/deploy.h @@ -46,7 +46,8 @@ struct TORCH_API InterpreterSession { // NOLINTNEXTLINE(bugprone-exception-escape) ~InterpreterSession(); - // global imports a python object from the specified module. + // `global` imports a python object from the specified module. + // Specifically `global` is analogous to "import `name` from `module`" in python. Obj global(const char* module, const char* name) { return impl_->global(module, name); } @@ -57,6 +58,8 @@ struct TORCH_API InterpreterSession { // InterpreterSession* I)' instead. We will have no backwards compatibility // guarentees for this function. ReplicatedObj createMovable(Obj obj); + + // Converts a `ReplicatedObj` to an `Obj` on this InterpreterSession. Obj fromMovable(const ReplicatedObj& obj); protected: @@ -73,6 +76,9 @@ struct TORCH_API InterpreterSession { std::function deconstruction_callback_ = nullptr; }; +// An `Interpreter` represents an invidual subinterpreter created by +// `torch::deploy`. It allows for the creation of `InterpreterSession` objects +// which allow users to interact with python objects. class TORCH_API Interpreter { private: void* handle_; @@ -84,10 +90,16 @@ class TORCH_API Interpreter { multipy::optional torchPluginFile_; public: + // Creates an Interpreter which is managed by `manager` and using the + // environment `env` Interpreter(InterpreterManager* manager, std::shared_ptr env); + + // Creates an Interpreter manager using environment `env` which is not tied to + // an Interpreter Manager. explicit Interpreter(std::shared_ptr env) : Interpreter(nullptr, env) {} + // Gets a new `InterpreterSession` from this Interpreter. InterpreterSession acquireSession() const { if (manager_) { return InterpreterSession(pImpl_->acquireSession(), manager_); @@ -95,6 +107,7 @@ class TORCH_API Interpreter { return InterpreterSession(pImpl_->acquireSession()); } } + ~Interpreter(); Interpreter(Interpreter&& rhs) noexcept : handle_(rhs.handle_), @@ -113,17 +126,28 @@ class TORCH_API Interpreter { struct Package; +// The default LoadBalancer for torch::deploy which handles allocating and +// freeing subinterpreters. struct TORCH_API LoadBalancer { + // Creates a Loadbalancer which handles `n` interpreters. explicit LoadBalancer(size_t n) : uses_(new uint64_t[8 * n]), allocated_(n), n_(n) { // 8*... to avoid false sharing of atomics on the same cache line memset(uses_.get(), 0, 8 * n_ * sizeof(uint64_t)); } + + // Changes the amount of subinterpreters which is handled by the load + // balancer. void setResourceLimit(size_t n) { MULTIPY_INTERNAL_ASSERT(n <= allocated_); n_ = n; } + + // Allocates an subinterpreter, and return its ID which is used to free it. int acquire(); + + // Frees the subinterpreter with ID `where`. This ID is returned by + // `LoadBalancer::acquire()` void free(int where); private: @@ -134,13 +158,19 @@ struct TORCH_API LoadBalancer { size_t n_; }; +// An `InterpreterManager` handles the interaction of multiple subinterpreters +// such as allocating subinterpreters, or load balancing the subinterpreters. struct TORCH_API InterpreterManager { + // constructor for `InterpreterManager` which takes the number of interpreters + // (usually correlates to number of cores on your cpu), and a pointer to an + // `Environment`. The default uses the local python env. explicit InterpreterManager( size_t nInterp = 2, std::shared_ptr env = std::make_shared()); - // get a free model, guarenteed that no other user of acquireOne has the same - // model. It _is_ possible that other users will be using the interpreter. + // Returns a free interpreter or an arbitrary interpreter if there are none free. + // To ensure data safety it's best to match the number of calling threads to the size of the interpreter + // pool to avoid sharing an interpreter. InterpreterSession acquireOne() { int where = resources_.acquire(); InterpreterSession I = instances_[where].acquireSession(); @@ -154,11 +184,19 @@ struct TORCH_API InterpreterManager { at::ArrayRef allInstances() { return instances_; } + + // debugging tool to control the size of the loadBalancer + // and change the number of interpreters on the fly void debugLimitInterpreters(size_t N) { AT_ASSERT(N <= instances_.size()); resources_.setResourceLimit(N); } + + // loads a package from a file with name `uri` Package loadPackage(const std::string& uri); + + // loads a package from a `PyTorchStreamReader` or any class other which uses + // `ReadAdapterInterface` Package loadPackage( std::shared_ptr reader); @@ -171,10 +209,12 @@ struct TORCH_API InterpreterManager { registeredModuleSource_[std::move(name)] = std::move(src); } - // Util function for debugging. + // Util function for debugging which outputs the number of registered modules. size_t countRegisteredModuleSources() { return registeredModuleSource_.size(); } + + // Converts `obj` from on `InterpreterSession` I into a `ReplicatedObj`. ReplicatedObj createMovable(Obj obj, InterpreterSession* I); InterpreterManager(const InterpreterManager&) = delete; InterpreterManager& operator=(const InterpreterManager&) = delete; @@ -204,8 +244,16 @@ struct TORCH_API ReplicatedObjImpl { InterpreterManager* manager_; }; +// ReplicatedObj represents a python object that can be used on multiple interpreters. Calling +// methods on this will pick an arbitrary interpreter to run on, transfer it there if not already +// and run the method. A replicated object can be converted to an interpreter specific `Obj` using +// `InterpreterSession::fromMovable(ReplicatedObj)` struct TORCH_API ReplicatedObj { + // Default constructor for `ReplicatedObj` ReplicatedObj() : pImpl_(nullptr) {} + + // Creates a new InterpreterSession on onThisInterpreter if specified else + // uses an arbitrary one from InteprreterManager. InterpreterSession acquireSession( const Interpreter* onThisInterpreter = nullptr) const; at::IValue operator()(at::ArrayRef args) const { @@ -213,6 +261,9 @@ struct TORCH_API ReplicatedObj { return I.self(args).toIValue(); } + // Invokes the Python function or class on an arbitrary interpreter with arguments + // given by the tuple args and named arguments given by the dictionary kwargs + // (equivalent to python's `__call__`). [[nodiscard]] at::IValue callKwargs( std::vector args, std::unordered_map kwargs) const { @@ -220,17 +271,28 @@ struct TORCH_API ReplicatedObj { return I.self.callKwargs(std::move(args), std::move(kwargs)).toIValue(); } + // Invokes the Python function or class on an arbitrary interpreter.with named arguments given by the + // dictionary kwargs (equivalent to python's `__call__`). [[nodiscard]] at::IValue callKwargs( std::unordered_map kwargs) const { auto I = acquireSession(); return I.self.callKwargs(std::move(kwargs)).toIValue(); } - [[nodiscard]] bool hasattr(const char* name) const { + // Returns true if `ReplicatedObj` has attribute with name `attr` and false + // otherwise. This is done on an arbitrary `InterpreterSession` which belongs + // to the `ReplicatedObj`'s manager. + [[nodiscard]] bool hasattr(const char* attr) const { auto I = acquireSession(); - return I.self.hasattr(name); + return I.self.hasattr(attr); } + + // Deletes `ReplicatedObj` from onThisInterpreter, if onThisInterpreter is + // `nullptr`, unload is called on all interpreters belonging to the + // ReplicatedObject's InterpreterManager void unload(const Interpreter* onThisInterpreter = nullptr); + + // Converts `ReplicatedObj` to `Obj` on `InterpreterSession` `I` Obj toObj(InterpreterSession* I); private: @@ -242,21 +304,24 @@ struct TORCH_API ReplicatedObj { friend struct InterpreterManager; }; +// PythonMethodWrapper is a more specific instance of a +// ReplicatedObj which represents a python method, and +// is therefore callable and has argument names accessible. class PythonMethodWrapper : public torch::IMethod { - // PythonMethodWrapper is a more specific instance of a - // ReplicatedObj which represents a python method, and - // is therefore callable and has argument names accessible. public: // TODO(whc) make bound method pickleable, then directly construct from that + PythonMethodWrapper( torch::deploy::ReplicatedObj model, std::string methodName) : model_(std::move(model)), methodName_(std::move(methodName)) {} + // return the name of the python method. const std::string& name() const override { return methodName_; } + // overrides the `()` operater to call the underlying python method. c10::IValue operator()( std::vector args, const IValueMap& kwargs = IValueMap()) const override { @@ -274,6 +339,8 @@ class PythonMethodWrapper : public torch::IMethod { std::string methodName_; }; +// Package is a wrapper around `torch.package` which allows loading a +// PyTorch model and its dependencies from a package. struct TORCH_API Package { // shorthand for getting the object as a pickle resource in the package ReplicatedObj loadPickle(const std::string& module, const std::string& file) { @@ -308,12 +375,16 @@ struct TORCH_API Package { } #endif + // Allocates an `InterpreterSession` and load the appropriate torch.package + // with it. InterpreterSession acquireSession() { auto I = manager_->acquireOne(); I.self = I.impl_->createOrGetPackageImporterFromContainerFile(containerFile_); return I; } + + // Converts an `Obj` from `InterpreterSession` `I` into a `ReplicatedObj`. ReplicatedObj createMovable(Obj obj, InterpreterSession* I) { return manager_->createMovable(obj, I); } diff --git a/multipy/runtime/elf_file.h b/multipy/runtime/elf_file.h index fdfd550c..46fe7d9d 100644 --- a/multipy/runtime/elf_file.h +++ b/multipy/runtime/elf_file.h @@ -16,6 +16,7 @@ namespace torch { namespace deploy { +// A representation of a section of an ElfFile. struct Section { Section() {} explicit Section( @@ -35,13 +36,18 @@ struct Section { } }; +// TODO: consolidate other ELF file related functions in loader.cpp to this file + /* * This class provie utilities to handle ELF file. Only support 64bit ELF file. */ -// TODO: consolidate other ELF file related functions in loader.cpp to this file class ElfFile { public: + // Constructs an Elffile with the corresponding `filename` explicit ElfFile(const char* filename); + + // Finds and return a `Section` with the corresponding `name`. If nothing is + // found, then a `multipy::nullopt` is returned. multipy::optional
findSection(const char* name) const; private: diff --git a/multipy/runtime/embedded_file.h b/multipy/runtime/embedded_file.h index 21b361b4..12680809 100644 --- a/multipy/runtime/embedded_file.h +++ b/multipy/runtime/embedded_file.h @@ -11,17 +11,20 @@ namespace torch { namespace deploy { +// Specifies which ELF section to load the interpreter from and the associated config. struct ExeSection { const char* sectionName; bool customLoader; }; +// Specifies which ELF symbols to load the interpreter from and the associated config. struct InterpreterSymbol { const char* startSym; const char* endSym; bool customLoader; }; +// EmbeddedFile makes it easier to load a custom interpreter embedded within the binary. struct EmbeddedFile { std::string libraryName{""}; bool customLoader{false}; diff --git a/multipy/runtime/environment.h b/multipy/runtime/environment.h index 3320616b..4b3906ed 100644 --- a/multipy/runtime/environment.h +++ b/multipy/runtime/environment.h @@ -42,6 +42,7 @@ class Environment { fclose(zippedFile); return zipArchive; } + void setupZippedPythonModules(const std::string& pythonAppDir) { #ifdef FBCODE_CAFFE2 extraPythonPaths_.push_back(getZippedArchive( @@ -56,14 +57,19 @@ class Environment { } public: + // Environment constructor which creates a random temporary directory as + // a directory for the zipped python modules. explicit Environment() { char tempDirName[] = "/tmp/torch_deploy_zipXXXXXX"; char* tempDirectory = mkdtemp(tempDirName); setupZippedPythonModules(tempDirectory); } + // Environment constructor which takes a file name for the + // directory for the zipped python modules. explicit Environment(const std::string& pythonAppDir) { setupZippedPythonModules(pythonAppDir); } + // Deconstructor for Environment. virtual ~Environment() { auto rmCmd = "rm -rf " + extraPythonLibrariesDir_; (void)system(rmCmd.c_str()); diff --git a/multipy/runtime/interpreter/interpreter_impl.h b/multipy/runtime/interpreter/interpreter_impl.h index 43cf70e2..b424cf72 100644 --- a/multipy/runtime/interpreter/interpreter_impl.h +++ b/multipy/runtime/interpreter/interpreter_impl.h @@ -19,6 +19,7 @@ namespace deploy { struct InterpreterSessionImpl; struct Obj; +// Representation a Pickled Object struct PickledObject { std::string data_; std::vector storages_; @@ -28,6 +29,8 @@ struct PickledObject { std::shared_ptr containerFile_; }; +// PickledObject contains a python object that's been pickled with the tensors saved separately. +// Unpickling this will share the underlying data across multiple copies/interpreters. struct InterpreterObj { friend struct Obj; friend struct ReplicatedObjImpl; @@ -72,14 +75,27 @@ struct Obj { : isDefault_(false), baseObj_(baseObj) {} Obj() : isDefault_(true), baseObj_(nullptr) {} + // return `IValue` representation. at::IValue toIValue() const; + + // Call an `Obj` callable, with arguments given by the tuple args. Equivalent to `__call__` in python. Obj operator()(at::ArrayRef args); + + // Call an `Obj` callable, with arguments given by the tuple args. Equivalent to `__call__` in python. Obj operator()(at::ArrayRef args); + + // Call an `Obj` callable, with arguments given by the tuple args, and named + // arguments given by the dictionary kwargs. Equivalent to `__call__` in python. Obj callKwargs( std::vector args, std::unordered_map kwargs); + // Call an `Obj` callable, with named arguments given by the dictionary + // kwargs. Equivalent to `__call__` in python. Obj callKwargs(std::unordered_map kwargs); + // Returns true if `Obj` has attribute with name `attr` and false otherwise. bool hasattr(const char* attr); + // Returns attribute `attr` from `Obj`. This is equivalent to calling + // `getattr(Obj, attr)` in python. Obj attr(const char* attr); private: @@ -87,6 +103,7 @@ struct Obj { std::shared_ptr baseObj_; }; +// The underlying implementation of `InterpreterSession` struct InterpreterSessionImpl { friend struct Package; friend struct ReplicatedObj; @@ -132,6 +149,7 @@ struct InterpreterSessionImpl { } }; +// The underlying implementation of `Interpreter` struct InterpreterImpl { virtual InterpreterSessionImpl* acquireSession() = 0; virtual void setFindModule( diff --git a/multipy/runtime/interpreter/plugin_registry.h b/multipy/runtime/interpreter/plugin_registry.h index 593a5cfc..97be6002 100644 --- a/multipy/runtime/interpreter/plugin_registry.h +++ b/multipy/runtime/interpreter/plugin_registry.h @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -11,21 +12,38 @@ namespace py = pybind11; namespace multipy { +// A `Converter` is used in order to convert `PyObject`s/`py::object` into +// an `IValue` or some other representation such as storage. class Converter { public: virtual ~Converter() = default; + // convert a `py::handle` to an `IValue` virtual multipy::optional toTypeInferredIValue( py::handle input) = 0; + + // convert an `IValue` into a `py::object` virtual multipy::optional toPyObject(at::IValue ivalue) = 0; + + // convert an `PyObject` into a `Storage` virtual multipy::optional createStorage(PyObject* obj) = 0; + + // create a `PyObject` from `storage` virtual multipy::optional createPyObject( const at::Storage& storage) = 0; + + // return the `THPDtype` of `scalarType` virtual multipy::optional getTHPDtype( at::ScalarType scalarType) = 0; }; +// register a converter to be used by torch::deploy / multipy. +// The order of the registration of the converters is dictated by the order of +// compilation. void registerConverter(Converter*); +// deregister a converter from torch::deploy / multipy +// The order of the deregistration of the converters is dictated by the order of +// compilation. void deregisterConverter(Converter*); at::IValue toTypeInferredIValue(py::handle input); diff --git a/multipy/runtime/mem_file.h b/multipy/runtime/mem_file.h index 404a3fbb..a68ff020 100644 --- a/multipy/runtime/mem_file.h +++ b/multipy/runtime/mem_file.h @@ -51,6 +51,8 @@ struct MemFile { [[nodiscard]] const char* data() const { return (const char*)mem_; } + + // return the file descriptor of the underlying file. int valid() { return fcntl(fd_, F_GETFD) != -1 || errno != EBADF; } @@ -62,6 +64,8 @@ struct MemFile { close(fd_); } } + + // return the size of the underlying file defined by the `MemFile` size_t size() { return n_bytes_; } diff --git a/multipy/runtime/noop_environment.h b/multipy/runtime/noop_environment.h index 4856f502..2891e29a 100644 --- a/multipy/runtime/noop_environment.h +++ b/multipy/runtime/noop_environment.h @@ -11,6 +11,7 @@ namespace torch { namespace deploy { +// The local python Environment class NoopEnvironment : public Environment { public: void configureInterpreter(Interpreter* /* interp */) override {} diff --git a/multipy/runtime/path_environment.h b/multipy/runtime/path_environment.h index 268f8dc6..06efaaf0 100644 --- a/multipy/runtime/path_environment.h +++ b/multipy/runtime/path_environment.h @@ -12,6 +12,8 @@ namespace torch { namespace deploy { +// An Environment which is defined by a specific path to python code (ie. condas +// sitepackages) class PathEnvironment : public Environment { public: explicit PathEnvironment(std::string path) : path_(std::move(path)) {}