From 390fb1a46fb7a4ab73bf8c3658895309f19ac5ff Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Thu, 4 Aug 2016 04:25:11 +0200 Subject: [PATCH 01/34] Fix custom eigen path in SConstruct --- SConstruct | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SConstruct b/SConstruct index 38cc781..e43de1b 100644 --- a/SConstruct +++ b/SConstruct @@ -85,7 +85,7 @@ if option("eigen", "") == "": /usr/local/include/eigen3 /usr/include/eigen3""".split()) else: - inc = findonpath("Eigen/Eigen", [option("eigen")]) + inc = findonpath("Eigen/Eigen", [option("eigen", "")]) env.Append(CPPPATH=[inc]) env.Append(LIBS=["png", "protobuf"]) From b9b0769a44de244e875fbccd67e759a7e1c584bd Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Thu, 4 Aug 2016 04:26:25 +0200 Subject: [PATCH 02/34] Fix isnan --- batches.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batches.cc b/batches.cc index 507ad40..1a26c02 100644 --- a/batches.cc +++ b/batches.cc @@ -76,7 +76,7 @@ bool anynan(TensorMap2 a) { for (int j = 0; j < a.dimension(0); j++) { for (int k = 0; k < a.dimension(1); k++) { float x = a(j, k); - if (isnan(x)) return true; + if (std::isnan(x)) return true; } } return false; From bdd149c32461480e3d809896b8ce61702e3f80b1 Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Thu, 4 Aug 2016 20:25:43 +0200 Subject: [PATCH 03/34] Fix test error reporting --- clstmocrtrain.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clstmocrtrain.cc b/clstmocrtrain.cc index eb31249..f59b99a 100644 --- a/clstmocrtrain.cc +++ b/clstmocrtrain.cc @@ -164,8 +164,8 @@ int main1(int argc, char **argv) { if (test_trigger(trial)) { auto tse = test_set_error(clstm, testset); - double errors = tse.first; - double count = tse.second; + double count = tse.first; + double errors = tse.second; test_error = errors / count; print("ERROR", trial, test_error, " ", errors, count); if (test_error < best_error) { From c85894e3683befefa096468b2daa4c72266d6567 Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Sat, 6 Aug 2016 01:36:31 +0200 Subject: [PATCH 04/34] First running cythonized bindings --- _clstm.pxd | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++ pyclstm.pyx | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 75 +++++++++++++++++++++-------------------------------- 3 files changed, 173 insertions(+), 45 deletions(-) create mode 100644 _clstm.pxd create mode 100644 pyclstm.pyx diff --git a/_clstm.pxd b/_clstm.pxd new file mode 100644 index 0000000..ba2a4b7 --- /dev/null +++ b/_clstm.pxd @@ -0,0 +1,71 @@ +from libc.stddef cimport wchar_t +from libcpp.vector cimport vector +from libcpp.string cimport string +from libcpp.memory cimport shared_ptr + + +cdef extern from "" namespace "std": + cppclass wstring: + cppclass iterator: + iterator() + wchar_t* operator*() + iterator(iterator &) + iterator operator++() + iterator operator--() + iterator operator==(iterator) + iterator operator!=(iterator) + iterator begin() + iterator end() + + +cdef extern from "pstring.h": + wstring utf8_to_utf32(string s) + + +cdef extern from "clstm.h" namespace "ocropus": + cdef cppclass Assoc: + Assoc() + Assoc(string &s) + bint contains(string &key, bint parent = true) + string get(string &key) + string get(string &key, string default) + void set(string &key, string value) + + cdef cppclass INetwork: + Assoc attr + + ctypedef shared_ptr[INetwork] Network + + +cdef extern from "tensor.h" namespace "ocropus": + cppclass TensorMap2: + pass + + cdef cppclass Tensor2: + int dims[2] + float *ptr + void resize(int i, int j) + void put(float val, int i, int j) + float get(int i, int j) + TensorMap2 map() + +cdef extern from "clstmhl.h" namespace "ocropus": + struct CharPrediction: + int i + int x + wchar_t c + float p + + # NOTE: The content of `codec` should be the utf-32 characters that the + # network is supposed to learn, encoded as integers + cppclass CLSTMOCR: + int target_height + Network net + bint maybe_load(string &fname) + bint maybe_save(string &fname) + void createBidi(vector[int] codec, int nhidden) + void setLearningRate(float learning_rate, float momentum) + string train_utf8(TensorMap2 imgdata, string &target) + string predict_utf8(TensorMap2 imgdata) + void predict(vector[CharPrediction] &preds, TensorMap2 imgdata) + string aligned_utf8() diff --git a/pyclstm.pyx b/pyclstm.pyx new file mode 100644 index 0000000..3aae3c5 --- /dev/null +++ b/pyclstm.pyx @@ -0,0 +1,72 @@ +from cpython.ref cimport PyObject +from libc.stddef cimport wchar_t +from libc.stdlib cimport malloc, free +from libcpp.vector cimport vector +from libcpp.set cimport set + +cimport _clstm + +cdef extern from "Python.h": + ctypedef PyObject PyUnicodeObject + Py_ssize_t PyUnicode_AsWideChar(PyUnicodeObject *o, wchar_t *w, + Py_ssize_t size) + + +#cdef make_codec(texts): +# cdef set[int] codes +# cdef _clstm.wstring s +# for t in texts: +# s = _clstm.utf8_to_utf32(t.encode('utf8')) +# for c in s: +## codes.insert(c) +# cdef vector[int] codec +# for c in codes: +# codec.push_back(c) +# return codes + + +cdef class ClstmOcr: + cdef _clstm.CLSTMOCR *_ocr + + def __cinit__(self, str fname=None): + self._ocr = new _clstm.CLSTMOCR() + + cpdef load(self, str fname): + cdef bint rv = self._ocr.maybe_load(fname) + if not rv: + raise IOError("Could not load model from {}".format(fname)) + + cpdef save(self, str fname): + cdef bint rv = self._ocr.maybe_save(fname) + if not rv: + raise IOError("Could not save model to {}".format(fname)) + + cpdef create_bidi(self, chars, int num_hidden): + chars_str = u"".join(sorted(chars)) + cdef vector[int] codec + cdef Py_ssize_t length = len(chars_str.encode("UTF-16")) // 2 + cdef wchar_t *wchars = malloc(length * sizeof(wchar_t)) + cdef Py_ssize_t number_written = PyUnicode_AsWideChar( + chars_str, wchars, length) + codec.push_back(0) + for i in range(length-1): + codec.push_back((wchars[i])) + self._ocr.createBidi(codec, num_hidden) + + cpdef set_learning_rate(self, float learning_rate, float momentum): + self._ocr.setLearningRate(learning_rate, momentum) + + cpdef unicode train(self, img, unicode text): + cdef _clstm.Tensor2 data + data.resize(img.height, img.width) + for i in range(img.height): + for j in range(img.width): + px = img.getpixel((j, i)) + if isinstance(px, tuple): + px = sum(px)/len(px) + data.put(px, i, j) + return self._ocr.train_utf8( + data.map(), text.encode('utf8')).decode('utf8') + + def __dealloc__(self): + del self._ocr diff --git a/setup.py b/setup.py index 80085f3..1cfe5c3 100644 --- a/setup.py +++ b/setup.py @@ -1,51 +1,36 @@ -#!/usr/bin/env python +import os -import os, os.path from distutils.core import setup, Extension -from numpy.distutils.misc_util import get_numpy_include_dirs -from os.path import getctime +from Cython.Build import cythonize -import distutils.sysconfig -config = distutils.sysconfig.get_config_vars() -# OPT=-DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -# CFLAGS -for k,v in config.items(): - if type(v)==str and "-W" in v: - print k,v -def remove_warnings(opt): - opt = opt.split() - opt = [s for s in opt if not s.startswith("-W")] - return " ".join(opt) -config["OPT"] = remove_warnings(config["OPT"]) -config["CFLAGS"] = remove_warnings(config["CFLAGS"]) -config["CONFIGURE_CFLAGS"] = remove_warnings(config["CONFIGURE_CFLAGS"]) -config["LDSHARED"] = remove_warnings(config["LDSHARED"]) -# hgversion = os.popen("hg -q id").read().strip() -hgversion = "unknown" +def ensure_protobuf(): + exists = os.path.exists("clstm.pb.cc") + stale = os.path.getctime("clstm.pb.cc") < os.path.getctime("clstm.proto") + if not exists or stale: + print "Generating proto file" + os.system("protoc clstm.proto --cpp_out=.") -include_dirs = ['/usr/include/eigen3', '/usr/local/include/eigen3', '/usr/local/include', '/usr/include/hdf5/serial'] + get_numpy_include_dirs() -swig_opts = ["-c++"] + ["-I" + d for d in include_dirs] -swiglib = os.popen("swig -swiglib").read()[:-1] +ext = Extension( + "pyclstm", + sources=['pyclstm.pyx', 'clstm.cc', 'clstm_prefab.cc', 'extras.cc', + 'batches.cc', 'ctc.cc', 'clstm_proto.cc', 'clstm.pb.cc', + 'clstm_compute.cc', 'tensor.cc'], + include_dirs=['/usr/include/eigen3', '/usr/local/include/eigen3', + '/usr/local/include', '/usr/include/hdf5/serial', + '/usr/include/hdf5'], + libraries=['protobuf', 'png'], + language='c++', + extra_compile_args=['-std=c++11', '-Wno-unused-result', '-g', '-Ofast', + '-DNODISPLAY=1', '-DTHROW=throw', '-DNDEBUG', + '-DEIGEN_NO_DEBUG', '-finline', '-ffast-math', + '-fno-signaling-nans', '-funsafe-math-optimizations', + '-ffinite-math-only', '-march=native']) -if not os.path.exists("clstm.pb.cc") or \ - getctime("clstm.pb.cc") < getctime("clstm.proto"): - print "making proto file" - os.system("protoc clstm.proto --cpp_out=.") - -clstm = Extension('_clstm', - libraries = ['png','protobuf'], - swig_opts = swig_opts, - include_dirs = include_dirs, - extra_compile_args = ['-std=c++11','-Wno-sign-compare', - '-Dadd_raw=add','-DNODISPLAY=1','-DTHROW=throw', - '-DHGVERSION="\\"'+hgversion+'\\""'], - sources=['clstm.i','clstm.cc','clstm_prefab.cc','extras.cc', - 'ctc.cc','clstm_proto.cc','clstm.pb.cc']) - -setup (name = 'clstm', - version = '0.0', - author = "Thomas Breuel", - description = """clstm library bindings""", - ext_modules = [clstm], - py_modules = ["clstm"]) +ensure_protobuf() +setup( + name='clstm', + version='0.1', + author="Thomas Breuel, Johannes Baiter", + description="CLSTM Python bindings", + ext_modules=cythonize([ext])) From 22570e338760fcb9d3e4465b34fc40a8989ee5ec Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Sat, 6 Aug 2016 16:15:44 +0200 Subject: [PATCH 05/34] Implement bindings for recognition --- pyclstm.pyx | 62 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/pyclstm.pyx b/pyclstm.pyx index 3aae3c5..e8c0084 100644 --- a/pyclstm.pyx +++ b/pyclstm.pyx @@ -6,23 +6,31 @@ from libcpp.set cimport set cimport _clstm +from collections import namedtuple + + +CharPrediction = namedtuple("CharPrediction", + ("x_position", "char", "confidence")) + + cdef extern from "Python.h": ctypedef PyObject PyUnicodeObject Py_ssize_t PyUnicode_AsWideChar(PyUnicodeObject *o, wchar_t *w, Py_ssize_t size) + PyObject* PyUnicode_FromWideChar(wchar_t *w, Py_ssize_t size) -#cdef make_codec(texts): -# cdef set[int] codes -# cdef _clstm.wstring s -# for t in texts: -# s = _clstm.utf8_to_utf32(t.encode('utf8')) -# for c in s: -## codes.insert(c) -# cdef vector[int] codec -# for c in codes: -# codec.push_back(c) -# return codes +cdef load_img(img, _clstm.Tensor2 *data): + data.resize(img.width, img.height) + for i in range(img.height): + for j in range(img.width): + px = img.getpixel((j, i)) + if isinstance(px, tuple): + px = (sum(px)/len(px))/255. + else: + px = px/255. + px = -px + 1. + data[0].put(px, j, i) cdef class ClstmOcr: @@ -56,17 +64,33 @@ cdef class ClstmOcr: cpdef set_learning_rate(self, float learning_rate, float momentum): self._ocr.setLearningRate(learning_rate, momentum) - cpdef unicode train(self, img, unicode text): + def aligned(self): + return self._ocr.aligned_utf8() + + def train(self, img, unicode text): cdef _clstm.Tensor2 data - data.resize(img.height, img.width) - for i in range(img.height): - for j in range(img.width): - px = img.getpixel((j, i)) - if isinstance(px, tuple): - px = sum(px)/len(px) - data.put(px, i, j) + load_img(img, &data) return self._ocr.train_utf8( data.map(), text.encode('utf8')).decode('utf8') + def recognize(self, img): + cdef _clstm.Tensor2 data + load_img(img, &data) + return self._ocr.predict_utf8(data.map()).decode('utf8') + + def recognize_chars(self, img): + cdef _clstm.Tensor2 data + cdef vector[_clstm.CharPrediction] preds + cdef vector[_clstm.CharPrediction].iterator pred_it + cdef wchar_t[2] cur_char + load_img(img, &data) + self._ocr.predict(preds, data.map()) + for i in range(preds.size()): + cur_char[0] = preds[i].c + yield CharPrediction( + preds[i].x, + PyUnicode_FromWideChar(cur_char, 1), + preds[i].p) + def __dealloc__(self): del self._ocr From b8fe1bed2a952edb29936f52d8da8d176836cd03 Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Sat, 6 Aug 2016 17:00:40 +0200 Subject: [PATCH 06/34] Add levenshtein binding --- _clstm.pxd | 4 ++++ pyclstm.pyx | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/_clstm.pxd b/_clstm.pxd index ba2a4b7..b96acdc 100644 --- a/_clstm.pxd +++ b/_clstm.pxd @@ -22,6 +22,10 @@ cdef extern from "pstring.h": wstring utf8_to_utf32(string s) +cdef extern from "clstm.h": + cdef double levenshtein[A, B](A a, B b) + + cdef extern from "clstm.h" namespace "ocropus": cdef cppclass Assoc: Assoc() diff --git a/pyclstm.pyx b/pyclstm.pyx index e8c0084..b59362f 100644 --- a/pyclstm.pyx +++ b/pyclstm.pyx @@ -3,6 +3,7 @@ from libc.stddef cimport wchar_t from libc.stdlib cimport malloc, free from libcpp.vector cimport vector from libcpp.set cimport set +from libcpp.string cimport string cimport _clstm @@ -20,6 +21,11 @@ cdef extern from "Python.h": PyObject* PyUnicode_FromWideChar(wchar_t *w, Py_ssize_t size) +cpdef double levenshtein(unicode a, unicode b): + return _clstm.levenshtein[string, string]( + a.encode('utf8'), b.encode('utf8')) + + cdef load_img(img, _clstm.Tensor2 *data): data.resize(img.width, img.height) for i in range(img.height): From ec56f6d9ebecf4b3dd6cfa2e1522a97643acb3f5 Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Sat, 6 Aug 2016 17:00:56 +0200 Subject: [PATCH 07/34] Minor stuff - Use PIL.PyAccess when filling Tensor2 from image - Return unicode string from `ClstmOcr.aligned` - Disable warnings during compilation --- pyclstm.pyx | 18 +++++++++++++----- setup.py | 8 ++++---- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/pyclstm.pyx b/pyclstm.pyx index b59362f..2c5436b 100644 --- a/pyclstm.pyx +++ b/pyclstm.pyx @@ -27,16 +27,24 @@ cpdef double levenshtein(unicode a, unicode b): cdef load_img(img, _clstm.Tensor2 *data): + """ Copy image data from a PIL image to an Eigen tensor. + + :param img: Image + :type img: :py:class:`PIL.Image.Image` + """ data.resize(img.width, img.height) - for i in range(img.height): - for j in range(img.width): - px = img.getpixel((j, i)) + imgdata = img.load() + for i in range(img.width): + for j in range(img.height): + px = imgdata[i, j] + # Pillow returns pixels as [0, 255], but we need [0, 1] if isinstance(px, tuple): + # For color images, we use the mean across all channels px = (sum(px)/len(px))/255. else: px = px/255. px = -px + 1. - data[0].put(px, j, i) + data[0].put(px, i, j) cdef class ClstmOcr: @@ -71,7 +79,7 @@ cdef class ClstmOcr: self._ocr.setLearningRate(learning_rate, momentum) def aligned(self): - return self._ocr.aligned_utf8() + return self._ocr.aligned_utf8().decode('utf8') def train(self, img, unicode text): cdef _clstm.Tensor2 data diff --git a/setup.py b/setup.py index 1cfe5c3..9a99edd 100644 --- a/setup.py +++ b/setup.py @@ -17,12 +17,12 @@ def ensure_protobuf(): 'batches.cc', 'ctc.cc', 'clstm_proto.cc', 'clstm.pb.cc', 'clstm_compute.cc', 'tensor.cc'], include_dirs=['/usr/include/eigen3', '/usr/local/include/eigen3', - '/usr/local/include', '/usr/include/hdf5/serial', - '/usr/include/hdf5'], + '/usr/local/include', '/usr/include', + '/usr/include/hdf5/serial', '/usr/include/hdf5'], libraries=['protobuf', 'png'], language='c++', - extra_compile_args=['-std=c++11', '-Wno-unused-result', '-g', '-Ofast', - '-DNODISPLAY=1', '-DTHROW=throw', '-DNDEBUG', + extra_compile_args=['-w', '-std=c++11', '-Wno-unused-result', '-g', + '-DNODISPLAY=1', '-DTHROW=throw', '-DNDEBUG', '-Ofast', '-DEIGEN_NO_DEBUG', '-finline', '-ffast-math', '-fno-signaling-nans', '-funsafe-math-optimizations', '-ffinite-math-only', '-march=native']) From fdb0fc8f11c196f5a34663f50424853c6783a7e9 Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Sat, 6 Aug 2016 18:28:25 +0200 Subject: [PATCH 08/34] Add example Python script for UW3 training --- run_uw3_500.py | 43 +++++++++++++++++++++++++++++++++++++++++++ setup.py | 1 + 2 files changed, 44 insertions(+) create mode 100644 run_uw3_500.py diff --git a/run_uw3_500.py b/run_uw3_500.py new file mode 100644 index 0000000..43db75a --- /dev/null +++ b/run_uw3_500.py @@ -0,0 +1,43 @@ +from __future__ import division + +import glob +import random +from itertools import chain + +import pyclstm +from PIL import Image + +all_imgs = [Image.open(p) for p in sorted(glob.glob("./book/*/*.png"))] +all_texts = [open(p).read().strip().decode('utf8') + for p in sorted(glob.glob("./book/*/*.gt.txt"))] +all_data = zip(all_imgs, all_texts) + +train_data = all_data[:400] +test_data = all_data[400:] + +ocr = pyclstm.ClstmOcr() +ocr.create_bidi(set(chain.from_iterable(all_texts)), 100) +ocr.set_learning_rate(0.0001, 0.9) + +for i in range(2000000): + best_error = 1. + img, txt = random.choice(train_data) + out = ocr.train(img, txt) + if not i % 10: + aligned = ocr.aligned() + print("Truth: {}".format(txt)) + print("Aligned: {}".format(aligned)) + print("Output: {}".format(out)) + if not i % 1000: + errors = 0 + chars = 0 + for img, txt in test_data: + out = ocr.recognize(img) + errors += pyclstm.levenshtein(txt, out) + chars += len(txt) + error = errors / chars + print ("=== Test set error after {} iterations: {:.2f}" + .format(i, error)) + if error < best_error: + print("=== New best error, saving model to model.clstm") + ocr.save("./model.clstm") diff --git a/setup.py b/setup.py index 9a99edd..1211308 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,7 @@ def ensure_protobuf(): print "Generating proto file" os.system("protoc clstm.proto --cpp_out=.") + ext = Extension( "pyclstm", sources=['pyclstm.pyx', 'clstm.cc', 'clstm_prefab.cc', 'extras.cc', From 2aed97cb854944bbffb57da42655829a439a1c62 Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Sun, 7 Aug 2016 11:33:10 +0200 Subject: [PATCH 09/34] Remove SWIG bindings --- clstm.i | 304 -------------------------------------------------------- 1 file changed, 304 deletions(-) delete mode 100644 clstm.i diff --git a/clstm.i b/clstm.i deleted file mode 100644 index 9fb45ce..0000000 --- a/clstm.i +++ /dev/null @@ -1,304 +0,0 @@ -// -*- C++ -*- - -%{ -#pragma GCC diagnostic ignored "-Wstrict-aliasing" -#pragma GCC diagnostic ignored "-Wuninitialized" -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" -#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION -%} - -%module(docstring="C-version of the ocropy LSTM implementation") clstm; -%feature("autodoc",1); -%include "typemaps.i" -%include "std_string.i" -%include "std_wstring.i" -%include "std_shared_ptr.i" -%include "std_vector.i" -%shared_ptr(ITrainable) -%shared_ptr(INetwork) -#ifdef SWIGPYTHON -%include "cstring.i" -#endif - -%{ -#include -#include -#include "clstm.h" -#include "clstm_compute.h" -using namespace ocropus; -using namespace std; -%} - -typedef float Float; -using std::string; - -#ifdef SWIGPYTHON -%exception { - try { - $action - } - catch(const char *s) { - PyErr_SetString(PyExc_IndexError,s); - return NULL; - } - catch(...) { - PyErr_SetString(PyExc_IndexError,"unknown exception in iulib"); - return NULL; - } -} -#endif - -%{ -#include "numpy/arrayobject.h" -%} - -%init %{ -import_array(); -%} - -/* create simple interface definitions for the built-in Sequence types */ - -struct Classes { - Classes(); - ~Classes(); - %rename(__getitem__) operator[]; - int operator[](int i); - int size(); - void resize(int); -}; -%extend Classes { - void __setitem__(int i,int value) { - (*$self)[i] = value; - } -} - -struct Batch { - void resize(int,int); - void setZero(int,int); - int rows(); - int cols(); - float &v(int,int); - float &d(int,int); -}; - -struct Params { - void resize(int,int); - void setZero(int,int); - int rows(); - int cols(); - float &v(int,int); - float &d(int,int); -}; - - -struct Sequence { - Sequence(); - ~Sequence(); - int size(); - int rows(); - int cols(); - %rename(__getitem__) operator[]; - Batch &operator[](int i); -}; - -struct Assoc { - string get(string key); - string get(string key, string dflt); - void set(string key, string dflt); -}; - -struct Codec { - std::vector codec; - int size() { return codec.size(); } - void set(const vector &data); - wchar_t decode(int cls); - std::wstring decode(Classes &cs); - void encode(Classes &cs, const std::wstring &s); -private: - void operator=(const Codec &); -}; - -struct INetwork; -typedef std::shared_ptr Network; -%template(vectornet) std::vector >; - -struct INetwork { - string kind; - Assoc attr; - virtual void setLearningRate(Float lr, Float momentum) = 0; - virtual void forward() = 0; - virtual void backward() = 0; - virtual void initialize(); - virtual ~INetwork(); - Sequence inputs; - Sequence outputs; - std::vector > sub; - Codec codec; - Codec icodec; - virtual int ninput(); - virtual int noutput(); - virtual void add(std::shared_ptr net); -}; - -void sgd_update(Network net); -void set_inputs(Network net, Sequence &inputs); -void set_targets(Network net, Sequence &targets); -void set_classes(Network net, Classes &classes); -void mktargets(Sequence &seq, Classes &targets, int ndim); - -std::shared_ptr make_layer(string); -std::shared_ptr make_net_init(string,string); - -#if 0 -%rename(seq_forward) forward_algorithm; -void forward_algorithm(Mat &lr,Mat &lmatch,double skip=-5.0); -%rename(seq_forwardbackward) forwardbackward; -void forwardbackward(Mat &both,Mat &lmatch); -#endif - -%rename(seq_ctc_align) ctc_align_targets; -void ctc_align_targets(Sequence &posteriors,Sequence &outputs,Sequence &targets); -void mktargets(Sequence &seq, Classes &targets, int ndim); - -void save_net(const string &file, Network net); -Network load_net(const string &file); - -%rename(network_info) network_info_as_strings; - -%inline %{ -int string_edit_distance(string a, string b) { - return levenshtein(a, b); -} - -string network_info_as_string(Network net) { - string result = ""; - walk_networks(net, [&result] (string s, INetwork *net) { - double lr = net->attr.get("learning_rate","-1"); - double momentum = net->attr.get("momentum","-1"); - result += s + ": " + to_string(lr); - result += string(" ") + to_string(momentum); - result += string(" ") + to_string(net->ninput()); - result += string(" ") + to_string(net->noutput()); - result += "\n"; - }); - return result; -} - -string sequence_info(Sequence &seq) { - string result = ""; - result += to_string(seq.size()); - result += string(":") + (seq.size()>0?to_string(seq[0].rows()):"*"); - result += string(":") + (seq.size()>0?to_string(seq[0].cols()):"*"); -#if 0 - // FIXME - double lo = 1e99, hi = -1e99; - for (int t=0;t Date: Sun, 7 Aug 2016 11:52:41 +0200 Subject: [PATCH 10/34] Include compiled protobuf files --- clstm.pb.cc | 1407 +++++++++++++++++++++++++++++++++++++++++++++++++++ clstm.pb.h | 1102 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 2509 insertions(+) create mode 100644 clstm.pb.cc create mode 100644 clstm.pb.h diff --git a/clstm.pb.cc b/clstm.pb.cc new file mode 100644 index 0000000..e6c1ce7 --- /dev/null +++ b/clstm.pb.cc @@ -0,0 +1,1407 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: clstm.proto + +#define INTERNAL_SUPPRESS_PROTOBUF_FIELD_DEPRECATION +#include "clstm.pb.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +// @@protoc_insertion_point(includes) + +namespace clstm { + +namespace { + +const ::google::protobuf::Descriptor* KeyValue_descriptor_ = NULL; +const ::google::protobuf::internal::GeneratedMessageReflection* + KeyValue_reflection_ = NULL; +const ::google::protobuf::Descriptor* Array_descriptor_ = NULL; +const ::google::protobuf::internal::GeneratedMessageReflection* + Array_reflection_ = NULL; +const ::google::protobuf::Descriptor* NetworkProto_descriptor_ = NULL; +const ::google::protobuf::internal::GeneratedMessageReflection* + NetworkProto_reflection_ = NULL; + +} // namespace + + +void protobuf_AssignDesc_clstm_2eproto() { + protobuf_AddDesc_clstm_2eproto(); + const ::google::protobuf::FileDescriptor* file = + ::google::protobuf::DescriptorPool::generated_pool()->FindFileByName( + "clstm.proto"); + GOOGLE_CHECK(file != NULL); + KeyValue_descriptor_ = file->message_type(0); + static const int KeyValue_offsets_[2] = { + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(KeyValue, key_), + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(KeyValue, value_), + }; + KeyValue_reflection_ = + new ::google::protobuf::internal::GeneratedMessageReflection( + KeyValue_descriptor_, + KeyValue::default_instance_, + KeyValue_offsets_, + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(KeyValue, _has_bits_[0]), + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(KeyValue, _unknown_fields_), + -1, + ::google::protobuf::DescriptorPool::generated_pool(), + ::google::protobuf::MessageFactory::generated_factory(), + sizeof(KeyValue)); + Array_descriptor_ = file->message_type(1); + static const int Array_offsets_[3] = { + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Array, name_), + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Array, dim_), + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Array, value_), + }; + Array_reflection_ = + new ::google::protobuf::internal::GeneratedMessageReflection( + Array_descriptor_, + Array::default_instance_, + Array_offsets_, + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Array, _has_bits_[0]), + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Array, _unknown_fields_), + -1, + ::google::protobuf::DescriptorPool::generated_pool(), + ::google::protobuf::MessageFactory::generated_factory(), + sizeof(Array)); + NetworkProto_descriptor_ = file->message_type(2); + static const int NetworkProto_offsets_[9] = { + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(NetworkProto, kind_), + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(NetworkProto, name_), + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(NetworkProto, ninput_), + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(NetworkProto, noutput_), + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(NetworkProto, icodec_), + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(NetworkProto, codec_), + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(NetworkProto, attribute_), + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(NetworkProto, weights_), + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(NetworkProto, sub_), + }; + NetworkProto_reflection_ = + new ::google::protobuf::internal::GeneratedMessageReflection( + NetworkProto_descriptor_, + NetworkProto::default_instance_, + NetworkProto_offsets_, + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(NetworkProto, _has_bits_[0]), + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(NetworkProto, _unknown_fields_), + -1, + ::google::protobuf::DescriptorPool::generated_pool(), + ::google::protobuf::MessageFactory::generated_factory(), + sizeof(NetworkProto)); +} + +namespace { + +GOOGLE_PROTOBUF_DECLARE_ONCE(protobuf_AssignDescriptors_once_); +inline void protobuf_AssignDescriptorsOnce() { + ::google::protobuf::GoogleOnceInit(&protobuf_AssignDescriptors_once_, + &protobuf_AssignDesc_clstm_2eproto); +} + +void protobuf_RegisterTypes(const ::std::string&) { + protobuf_AssignDescriptorsOnce(); + ::google::protobuf::MessageFactory::InternalRegisterGeneratedMessage( + KeyValue_descriptor_, &KeyValue::default_instance()); + ::google::protobuf::MessageFactory::InternalRegisterGeneratedMessage( + Array_descriptor_, &Array::default_instance()); + ::google::protobuf::MessageFactory::InternalRegisterGeneratedMessage( + NetworkProto_descriptor_, &NetworkProto::default_instance()); +} + +} // namespace + +void protobuf_ShutdownFile_clstm_2eproto() { + delete KeyValue::default_instance_; + delete KeyValue_reflection_; + delete Array::default_instance_; + delete Array_reflection_; + delete NetworkProto::default_instance_; + delete NetworkProto_reflection_; +} + +void protobuf_AddDesc_clstm_2eproto() { + static bool already_here = false; + if (already_here) return; + already_here = true; + GOOGLE_PROTOBUF_VERIFY_VERSION; + + ::google::protobuf::DescriptorPool::InternalAddGeneratedFile( + "\n\013clstm.proto\022\005clstm\"&\n\010KeyValue\022\013\n\003key\030" + "\001 \002(\t\022\r\n\005value\030\002 \002(\t\"1\n\005Array\022\014\n\004name\030\001 " + "\001(\t\022\013\n\003dim\030\002 \003(\005\022\r\n\005value\030\003 \003(\002\"\317\001\n\014Netw" + "orkProto\022\014\n\004kind\030\001 \002(\t\022\014\n\004name\030\002 \001(\t\022\016\n\006" + "ninput\030\n \002(\005\022\017\n\007noutput\030\013 \002(\005\022\016\n\006icodec\030" + "\014 \003(\005\022\r\n\005codec\030\r \003(\005\022\"\n\tattribute\030\024 \003(\0132" + "\017.clstm.KeyValue\022\035\n\007weights\030\036 \003(\0132\014.clst" + "m.Array\022 \n\003sub\030( \003(\0132\023.clstm.NetworkProt" + "o", 321); + ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile( + "clstm.proto", &protobuf_RegisterTypes); + KeyValue::default_instance_ = new KeyValue(); + Array::default_instance_ = new Array(); + NetworkProto::default_instance_ = new NetworkProto(); + KeyValue::default_instance_->InitAsDefaultInstance(); + Array::default_instance_->InitAsDefaultInstance(); + NetworkProto::default_instance_->InitAsDefaultInstance(); + ::google::protobuf::internal::OnShutdown(&protobuf_ShutdownFile_clstm_2eproto); +} + +// Force AddDescriptors() to be called at static initialization time. +struct StaticDescriptorInitializer_clstm_2eproto { + StaticDescriptorInitializer_clstm_2eproto() { + protobuf_AddDesc_clstm_2eproto(); + } +} static_descriptor_initializer_clstm_2eproto_; + +// =================================================================== + +#ifndef _MSC_VER +const int KeyValue::kKeyFieldNumber; +const int KeyValue::kValueFieldNumber; +#endif // !_MSC_VER + +KeyValue::KeyValue() + : ::google::protobuf::Message() { + SharedCtor(); + // @@protoc_insertion_point(constructor:clstm.KeyValue) +} + +void KeyValue::InitAsDefaultInstance() { +} + +KeyValue::KeyValue(const KeyValue& from) + : ::google::protobuf::Message() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:clstm.KeyValue) +} + +void KeyValue::SharedCtor() { + ::google::protobuf::internal::GetEmptyString(); + _cached_size_ = 0; + key_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + value_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +KeyValue::~KeyValue() { + // @@protoc_insertion_point(destructor:clstm.KeyValue) + SharedDtor(); +} + +void KeyValue::SharedDtor() { + if (key_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete key_; + } + if (value_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete value_; + } + if (this != default_instance_) { + } +} + +void KeyValue::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const ::google::protobuf::Descriptor* KeyValue::descriptor() { + protobuf_AssignDescriptorsOnce(); + return KeyValue_descriptor_; +} + +const KeyValue& KeyValue::default_instance() { + if (default_instance_ == NULL) protobuf_AddDesc_clstm_2eproto(); + return *default_instance_; +} + +KeyValue* KeyValue::default_instance_ = NULL; + +KeyValue* KeyValue::New() const { + return new KeyValue; +} + +void KeyValue::Clear() { + if (_has_bits_[0 / 32] & 3) { + if (has_key()) { + if (key_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + key_->clear(); + } + } + if (has_value()) { + if (value_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + value_->clear(); + } + } + } + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->Clear(); +} + +bool KeyValue::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + // @@protoc_insertion_point(parse_start:clstm.KeyValue) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // required string key = 1; + case 1: { + if (tag == 10) { + DO_(::google::protobuf::internal::WireFormatLite::ReadString( + input, this->mutable_key())); + ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( + this->key().data(), this->key().length(), + ::google::protobuf::internal::WireFormat::PARSE, + "key"); + } else { + goto handle_unusual; + } + if (input->ExpectTag(18)) goto parse_value; + break; + } + + // required string value = 2; + case 2: { + if (tag == 18) { + parse_value: + DO_(::google::protobuf::internal::WireFormatLite::ReadString( + input, this->mutable_value())); + ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( + this->value().data(), this->value().length(), + ::google::protobuf::internal::WireFormat::PARSE, + "value"); + } else { + goto handle_unusual; + } + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormat::SkipField( + input, tag, mutable_unknown_fields())); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:clstm.KeyValue) + return true; +failure: + // @@protoc_insertion_point(parse_failure:clstm.KeyValue) + return false; +#undef DO_ +} + +void KeyValue::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:clstm.KeyValue) + // required string key = 1; + if (has_key()) { + ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( + this->key().data(), this->key().length(), + ::google::protobuf::internal::WireFormat::SERIALIZE, + "key"); + ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( + 1, this->key(), output); + } + + // required string value = 2; + if (has_value()) { + ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( + this->value().data(), this->value().length(), + ::google::protobuf::internal::WireFormat::SERIALIZE, + "value"); + ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( + 2, this->value(), output); + } + + if (!unknown_fields().empty()) { + ::google::protobuf::internal::WireFormat::SerializeUnknownFields( + unknown_fields(), output); + } + // @@protoc_insertion_point(serialize_end:clstm.KeyValue) +} + +::google::protobuf::uint8* KeyValue::SerializeWithCachedSizesToArray( + ::google::protobuf::uint8* target) const { + // @@protoc_insertion_point(serialize_to_array_start:clstm.KeyValue) + // required string key = 1; + if (has_key()) { + ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( + this->key().data(), this->key().length(), + ::google::protobuf::internal::WireFormat::SERIALIZE, + "key"); + target = + ::google::protobuf::internal::WireFormatLite::WriteStringToArray( + 1, this->key(), target); + } + + // required string value = 2; + if (has_value()) { + ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( + this->value().data(), this->value().length(), + ::google::protobuf::internal::WireFormat::SERIALIZE, + "value"); + target = + ::google::protobuf::internal::WireFormatLite::WriteStringToArray( + 2, this->value(), target); + } + + if (!unknown_fields().empty()) { + target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray( + unknown_fields(), target); + } + // @@protoc_insertion_point(serialize_to_array_end:clstm.KeyValue) + return target; +} + +int KeyValue::ByteSize() const { + int total_size = 0; + + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + // required string key = 1; + if (has_key()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::StringSize( + this->key()); + } + + // required string value = 2; + if (has_value()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::StringSize( + this->value()); + } + + } + if (!unknown_fields().empty()) { + total_size += + ::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize( + unknown_fields()); + } + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void KeyValue::MergeFrom(const ::google::protobuf::Message& from) { + GOOGLE_CHECK_NE(&from, this); + const KeyValue* source = + ::google::protobuf::internal::dynamic_cast_if_available( + &from); + if (source == NULL) { + ::google::protobuf::internal::ReflectionOps::Merge(from, this); + } else { + MergeFrom(*source); + } +} + +void KeyValue::MergeFrom(const KeyValue& from) { + GOOGLE_CHECK_NE(&from, this); + if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (from.has_key()) { + set_key(from.key()); + } + if (from.has_value()) { + set_value(from.value()); + } + } + mutable_unknown_fields()->MergeFrom(from.unknown_fields()); +} + +void KeyValue::CopyFrom(const ::google::protobuf::Message& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +void KeyValue::CopyFrom(const KeyValue& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool KeyValue::IsInitialized() const { + if ((_has_bits_[0] & 0x00000003) != 0x00000003) return false; + + return true; +} + +void KeyValue::Swap(KeyValue* other) { + if (other != this) { + std::swap(key_, other->key_); + std::swap(value_, other->value_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.Swap(&other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::google::protobuf::Metadata KeyValue::GetMetadata() const { + protobuf_AssignDescriptorsOnce(); + ::google::protobuf::Metadata metadata; + metadata.descriptor = KeyValue_descriptor_; + metadata.reflection = KeyValue_reflection_; + return metadata; +} + + +// =================================================================== + +#ifndef _MSC_VER +const int Array::kNameFieldNumber; +const int Array::kDimFieldNumber; +const int Array::kValueFieldNumber; +#endif // !_MSC_VER + +Array::Array() + : ::google::protobuf::Message() { + SharedCtor(); + // @@protoc_insertion_point(constructor:clstm.Array) +} + +void Array::InitAsDefaultInstance() { +} + +Array::Array(const Array& from) + : ::google::protobuf::Message() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:clstm.Array) +} + +void Array::SharedCtor() { + ::google::protobuf::internal::GetEmptyString(); + _cached_size_ = 0; + name_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +Array::~Array() { + // @@protoc_insertion_point(destructor:clstm.Array) + SharedDtor(); +} + +void Array::SharedDtor() { + if (name_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete name_; + } + if (this != default_instance_) { + } +} + +void Array::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const ::google::protobuf::Descriptor* Array::descriptor() { + protobuf_AssignDescriptorsOnce(); + return Array_descriptor_; +} + +const Array& Array::default_instance() { + if (default_instance_ == NULL) protobuf_AddDesc_clstm_2eproto(); + return *default_instance_; +} + +Array* Array::default_instance_ = NULL; + +Array* Array::New() const { + return new Array; +} + +void Array::Clear() { + if (has_name()) { + if (name_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + name_->clear(); + } + } + dim_.Clear(); + value_.Clear(); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->Clear(); +} + +bool Array::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + // @@protoc_insertion_point(parse_start:clstm.Array) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // optional string name = 1; + case 1: { + if (tag == 10) { + DO_(::google::protobuf::internal::WireFormatLite::ReadString( + input, this->mutable_name())); + ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( + this->name().data(), this->name().length(), + ::google::protobuf::internal::WireFormat::PARSE, + "name"); + } else { + goto handle_unusual; + } + if (input->ExpectTag(16)) goto parse_dim; + break; + } + + // repeated int32 dim = 2; + case 2: { + if (tag == 16) { + parse_dim: + DO_((::google::protobuf::internal::WireFormatLite::ReadRepeatedPrimitive< + ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( + 1, 16, input, this->mutable_dim()))); + } else if (tag == 18) { + DO_((::google::protobuf::internal::WireFormatLite::ReadPackedPrimitiveNoInline< + ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( + input, this->mutable_dim()))); + } else { + goto handle_unusual; + } + if (input->ExpectTag(16)) goto parse_dim; + if (input->ExpectTag(29)) goto parse_value; + break; + } + + // repeated float value = 3; + case 3: { + if (tag == 29) { + parse_value: + DO_((::google::protobuf::internal::WireFormatLite::ReadRepeatedPrimitive< + float, ::google::protobuf::internal::WireFormatLite::TYPE_FLOAT>( + 1, 29, input, this->mutable_value()))); + } else if (tag == 26) { + DO_((::google::protobuf::internal::WireFormatLite::ReadPackedPrimitiveNoInline< + float, ::google::protobuf::internal::WireFormatLite::TYPE_FLOAT>( + input, this->mutable_value()))); + } else { + goto handle_unusual; + } + if (input->ExpectTag(29)) goto parse_value; + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormat::SkipField( + input, tag, mutable_unknown_fields())); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:clstm.Array) + return true; +failure: + // @@protoc_insertion_point(parse_failure:clstm.Array) + return false; +#undef DO_ +} + +void Array::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:clstm.Array) + // optional string name = 1; + if (has_name()) { + ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( + this->name().data(), this->name().length(), + ::google::protobuf::internal::WireFormat::SERIALIZE, + "name"); + ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( + 1, this->name(), output); + } + + // repeated int32 dim = 2; + for (int i = 0; i < this->dim_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteInt32( + 2, this->dim(i), output); + } + + // repeated float value = 3; + for (int i = 0; i < this->value_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteFloat( + 3, this->value(i), output); + } + + if (!unknown_fields().empty()) { + ::google::protobuf::internal::WireFormat::SerializeUnknownFields( + unknown_fields(), output); + } + // @@protoc_insertion_point(serialize_end:clstm.Array) +} + +::google::protobuf::uint8* Array::SerializeWithCachedSizesToArray( + ::google::protobuf::uint8* target) const { + // @@protoc_insertion_point(serialize_to_array_start:clstm.Array) + // optional string name = 1; + if (has_name()) { + ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( + this->name().data(), this->name().length(), + ::google::protobuf::internal::WireFormat::SERIALIZE, + "name"); + target = + ::google::protobuf::internal::WireFormatLite::WriteStringToArray( + 1, this->name(), target); + } + + // repeated int32 dim = 2; + for (int i = 0; i < this->dim_size(); i++) { + target = ::google::protobuf::internal::WireFormatLite:: + WriteInt32ToArray(2, this->dim(i), target); + } + + // repeated float value = 3; + for (int i = 0; i < this->value_size(); i++) { + target = ::google::protobuf::internal::WireFormatLite:: + WriteFloatToArray(3, this->value(i), target); + } + + if (!unknown_fields().empty()) { + target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray( + unknown_fields(), target); + } + // @@protoc_insertion_point(serialize_to_array_end:clstm.Array) + return target; +} + +int Array::ByteSize() const { + int total_size = 0; + + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + // optional string name = 1; + if (has_name()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::StringSize( + this->name()); + } + + } + // repeated int32 dim = 2; + { + int data_size = 0; + for (int i = 0; i < this->dim_size(); i++) { + data_size += ::google::protobuf::internal::WireFormatLite:: + Int32Size(this->dim(i)); + } + total_size += 1 * this->dim_size() + data_size; + } + + // repeated float value = 3; + { + int data_size = 0; + data_size = 4 * this->value_size(); + total_size += 1 * this->value_size() + data_size; + } + + if (!unknown_fields().empty()) { + total_size += + ::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize( + unknown_fields()); + } + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void Array::MergeFrom(const ::google::protobuf::Message& from) { + GOOGLE_CHECK_NE(&from, this); + const Array* source = + ::google::protobuf::internal::dynamic_cast_if_available( + &from); + if (source == NULL) { + ::google::protobuf::internal::ReflectionOps::Merge(from, this); + } else { + MergeFrom(*source); + } +} + +void Array::MergeFrom(const Array& from) { + GOOGLE_CHECK_NE(&from, this); + dim_.MergeFrom(from.dim_); + value_.MergeFrom(from.value_); + if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (from.has_name()) { + set_name(from.name()); + } + } + mutable_unknown_fields()->MergeFrom(from.unknown_fields()); +} + +void Array::CopyFrom(const ::google::protobuf::Message& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +void Array::CopyFrom(const Array& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool Array::IsInitialized() const { + + return true; +} + +void Array::Swap(Array* other) { + if (other != this) { + std::swap(name_, other->name_); + dim_.Swap(&other->dim_); + value_.Swap(&other->value_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.Swap(&other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::google::protobuf::Metadata Array::GetMetadata() const { + protobuf_AssignDescriptorsOnce(); + ::google::protobuf::Metadata metadata; + metadata.descriptor = Array_descriptor_; + metadata.reflection = Array_reflection_; + return metadata; +} + + +// =================================================================== + +#ifndef _MSC_VER +const int NetworkProto::kKindFieldNumber; +const int NetworkProto::kNameFieldNumber; +const int NetworkProto::kNinputFieldNumber; +const int NetworkProto::kNoutputFieldNumber; +const int NetworkProto::kIcodecFieldNumber; +const int NetworkProto::kCodecFieldNumber; +const int NetworkProto::kAttributeFieldNumber; +const int NetworkProto::kWeightsFieldNumber; +const int NetworkProto::kSubFieldNumber; +#endif // !_MSC_VER + +NetworkProto::NetworkProto() + : ::google::protobuf::Message() { + SharedCtor(); + // @@protoc_insertion_point(constructor:clstm.NetworkProto) +} + +void NetworkProto::InitAsDefaultInstance() { +} + +NetworkProto::NetworkProto(const NetworkProto& from) + : ::google::protobuf::Message() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:clstm.NetworkProto) +} + +void NetworkProto::SharedCtor() { + ::google::protobuf::internal::GetEmptyString(); + _cached_size_ = 0; + kind_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + name_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + ninput_ = 0; + noutput_ = 0; + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +NetworkProto::~NetworkProto() { + // @@protoc_insertion_point(destructor:clstm.NetworkProto) + SharedDtor(); +} + +void NetworkProto::SharedDtor() { + if (kind_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete kind_; + } + if (name_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete name_; + } + if (this != default_instance_) { + } +} + +void NetworkProto::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const ::google::protobuf::Descriptor* NetworkProto::descriptor() { + protobuf_AssignDescriptorsOnce(); + return NetworkProto_descriptor_; +} + +const NetworkProto& NetworkProto::default_instance() { + if (default_instance_ == NULL) protobuf_AddDesc_clstm_2eproto(); + return *default_instance_; +} + +NetworkProto* NetworkProto::default_instance_ = NULL; + +NetworkProto* NetworkProto::New() const { + return new NetworkProto; +} + +void NetworkProto::Clear() { +#define OFFSET_OF_FIELD_(f) (reinterpret_cast( \ + &reinterpret_cast(16)->f) - \ + reinterpret_cast(16)) + +#define ZR_(first, last) do { \ + size_t f = OFFSET_OF_FIELD_(first); \ + size_t n = OFFSET_OF_FIELD_(last) - f + sizeof(last); \ + ::memset(&first, 0, n); \ + } while (0) + + if (_has_bits_[0 / 32] & 15) { + ZR_(ninput_, noutput_); + if (has_kind()) { + if (kind_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + kind_->clear(); + } + } + if (has_name()) { + if (name_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + name_->clear(); + } + } + } + +#undef OFFSET_OF_FIELD_ +#undef ZR_ + + icodec_.Clear(); + codec_.Clear(); + attribute_.Clear(); + weights_.Clear(); + sub_.Clear(); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->Clear(); +} + +bool NetworkProto::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + // @@protoc_insertion_point(parse_start:clstm.NetworkProto) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(16383); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // required string kind = 1; + case 1: { + if (tag == 10) { + DO_(::google::protobuf::internal::WireFormatLite::ReadString( + input, this->mutable_kind())); + ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( + this->kind().data(), this->kind().length(), + ::google::protobuf::internal::WireFormat::PARSE, + "kind"); + } else { + goto handle_unusual; + } + if (input->ExpectTag(18)) goto parse_name; + break; + } + + // optional string name = 2; + case 2: { + if (tag == 18) { + parse_name: + DO_(::google::protobuf::internal::WireFormatLite::ReadString( + input, this->mutable_name())); + ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( + this->name().data(), this->name().length(), + ::google::protobuf::internal::WireFormat::PARSE, + "name"); + } else { + goto handle_unusual; + } + if (input->ExpectTag(80)) goto parse_ninput; + break; + } + + // required int32 ninput = 10; + case 10: { + if (tag == 80) { + parse_ninput: + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( + input, &ninput_))); + set_has_ninput(); + } else { + goto handle_unusual; + } + if (input->ExpectTag(88)) goto parse_noutput; + break; + } + + // required int32 noutput = 11; + case 11: { + if (tag == 88) { + parse_noutput: + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( + input, &noutput_))); + set_has_noutput(); + } else { + goto handle_unusual; + } + if (input->ExpectTag(96)) goto parse_icodec; + break; + } + + // repeated int32 icodec = 12; + case 12: { + if (tag == 96) { + parse_icodec: + DO_((::google::protobuf::internal::WireFormatLite::ReadRepeatedPrimitive< + ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( + 1, 96, input, this->mutable_icodec()))); + } else if (tag == 98) { + DO_((::google::protobuf::internal::WireFormatLite::ReadPackedPrimitiveNoInline< + ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( + input, this->mutable_icodec()))); + } else { + goto handle_unusual; + } + if (input->ExpectTag(96)) goto parse_icodec; + if (input->ExpectTag(104)) goto parse_codec; + break; + } + + // repeated int32 codec = 13; + case 13: { + if (tag == 104) { + parse_codec: + DO_((::google::protobuf::internal::WireFormatLite::ReadRepeatedPrimitive< + ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( + 1, 104, input, this->mutable_codec()))); + } else if (tag == 106) { + DO_((::google::protobuf::internal::WireFormatLite::ReadPackedPrimitiveNoInline< + ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( + input, this->mutable_codec()))); + } else { + goto handle_unusual; + } + if (input->ExpectTag(104)) goto parse_codec; + if (input->ExpectTag(162)) goto parse_attribute; + break; + } + + // repeated .clstm.KeyValue attribute = 20; + case 20: { + if (tag == 162) { + parse_attribute: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, add_attribute())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(162)) goto parse_attribute; + if (input->ExpectTag(242)) goto parse_weights; + break; + } + + // repeated .clstm.Array weights = 30; + case 30: { + if (tag == 242) { + parse_weights: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, add_weights())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(242)) goto parse_weights; + if (input->ExpectTag(322)) goto parse_sub; + break; + } + + // repeated .clstm.NetworkProto sub = 40; + case 40: { + if (tag == 322) { + parse_sub: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, add_sub())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(322)) goto parse_sub; + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormat::SkipField( + input, tag, mutable_unknown_fields())); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:clstm.NetworkProto) + return true; +failure: + // @@protoc_insertion_point(parse_failure:clstm.NetworkProto) + return false; +#undef DO_ +} + +void NetworkProto::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:clstm.NetworkProto) + // required string kind = 1; + if (has_kind()) { + ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( + this->kind().data(), this->kind().length(), + ::google::protobuf::internal::WireFormat::SERIALIZE, + "kind"); + ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( + 1, this->kind(), output); + } + + // optional string name = 2; + if (has_name()) { + ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( + this->name().data(), this->name().length(), + ::google::protobuf::internal::WireFormat::SERIALIZE, + "name"); + ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( + 2, this->name(), output); + } + + // required int32 ninput = 10; + if (has_ninput()) { + ::google::protobuf::internal::WireFormatLite::WriteInt32(10, this->ninput(), output); + } + + // required int32 noutput = 11; + if (has_noutput()) { + ::google::protobuf::internal::WireFormatLite::WriteInt32(11, this->noutput(), output); + } + + // repeated int32 icodec = 12; + for (int i = 0; i < this->icodec_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteInt32( + 12, this->icodec(i), output); + } + + // repeated int32 codec = 13; + for (int i = 0; i < this->codec_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteInt32( + 13, this->codec(i), output); + } + + // repeated .clstm.KeyValue attribute = 20; + for (int i = 0; i < this->attribute_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray( + 20, this->attribute(i), output); + } + + // repeated .clstm.Array weights = 30; + for (int i = 0; i < this->weights_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray( + 30, this->weights(i), output); + } + + // repeated .clstm.NetworkProto sub = 40; + for (int i = 0; i < this->sub_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray( + 40, this->sub(i), output); + } + + if (!unknown_fields().empty()) { + ::google::protobuf::internal::WireFormat::SerializeUnknownFields( + unknown_fields(), output); + } + // @@protoc_insertion_point(serialize_end:clstm.NetworkProto) +} + +::google::protobuf::uint8* NetworkProto::SerializeWithCachedSizesToArray( + ::google::protobuf::uint8* target) const { + // @@protoc_insertion_point(serialize_to_array_start:clstm.NetworkProto) + // required string kind = 1; + if (has_kind()) { + ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( + this->kind().data(), this->kind().length(), + ::google::protobuf::internal::WireFormat::SERIALIZE, + "kind"); + target = + ::google::protobuf::internal::WireFormatLite::WriteStringToArray( + 1, this->kind(), target); + } + + // optional string name = 2; + if (has_name()) { + ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( + this->name().data(), this->name().length(), + ::google::protobuf::internal::WireFormat::SERIALIZE, + "name"); + target = + ::google::protobuf::internal::WireFormatLite::WriteStringToArray( + 2, this->name(), target); + } + + // required int32 ninput = 10; + if (has_ninput()) { + target = ::google::protobuf::internal::WireFormatLite::WriteInt32ToArray(10, this->ninput(), target); + } + + // required int32 noutput = 11; + if (has_noutput()) { + target = ::google::protobuf::internal::WireFormatLite::WriteInt32ToArray(11, this->noutput(), target); + } + + // repeated int32 icodec = 12; + for (int i = 0; i < this->icodec_size(); i++) { + target = ::google::protobuf::internal::WireFormatLite:: + WriteInt32ToArray(12, this->icodec(i), target); + } + + // repeated int32 codec = 13; + for (int i = 0; i < this->codec_size(); i++) { + target = ::google::protobuf::internal::WireFormatLite:: + WriteInt32ToArray(13, this->codec(i), target); + } + + // repeated .clstm.KeyValue attribute = 20; + for (int i = 0; i < this->attribute_size(); i++) { + target = ::google::protobuf::internal::WireFormatLite:: + WriteMessageNoVirtualToArray( + 20, this->attribute(i), target); + } + + // repeated .clstm.Array weights = 30; + for (int i = 0; i < this->weights_size(); i++) { + target = ::google::protobuf::internal::WireFormatLite:: + WriteMessageNoVirtualToArray( + 30, this->weights(i), target); + } + + // repeated .clstm.NetworkProto sub = 40; + for (int i = 0; i < this->sub_size(); i++) { + target = ::google::protobuf::internal::WireFormatLite:: + WriteMessageNoVirtualToArray( + 40, this->sub(i), target); + } + + if (!unknown_fields().empty()) { + target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray( + unknown_fields(), target); + } + // @@protoc_insertion_point(serialize_to_array_end:clstm.NetworkProto) + return target; +} + +int NetworkProto::ByteSize() const { + int total_size = 0; + + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + // required string kind = 1; + if (has_kind()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::StringSize( + this->kind()); + } + + // optional string name = 2; + if (has_name()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::StringSize( + this->name()); + } + + // required int32 ninput = 10; + if (has_ninput()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::Int32Size( + this->ninput()); + } + + // required int32 noutput = 11; + if (has_noutput()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::Int32Size( + this->noutput()); + } + + } + // repeated int32 icodec = 12; + { + int data_size = 0; + for (int i = 0; i < this->icodec_size(); i++) { + data_size += ::google::protobuf::internal::WireFormatLite:: + Int32Size(this->icodec(i)); + } + total_size += 1 * this->icodec_size() + data_size; + } + + // repeated int32 codec = 13; + { + int data_size = 0; + for (int i = 0; i < this->codec_size(); i++) { + data_size += ::google::protobuf::internal::WireFormatLite:: + Int32Size(this->codec(i)); + } + total_size += 1 * this->codec_size() + data_size; + } + + // repeated .clstm.KeyValue attribute = 20; + total_size += 2 * this->attribute_size(); + for (int i = 0; i < this->attribute_size(); i++) { + total_size += + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->attribute(i)); + } + + // repeated .clstm.Array weights = 30; + total_size += 2 * this->weights_size(); + for (int i = 0; i < this->weights_size(); i++) { + total_size += + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->weights(i)); + } + + // repeated .clstm.NetworkProto sub = 40; + total_size += 2 * this->sub_size(); + for (int i = 0; i < this->sub_size(); i++) { + total_size += + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->sub(i)); + } + + if (!unknown_fields().empty()) { + total_size += + ::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize( + unknown_fields()); + } + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void NetworkProto::MergeFrom(const ::google::protobuf::Message& from) { + GOOGLE_CHECK_NE(&from, this); + const NetworkProto* source = + ::google::protobuf::internal::dynamic_cast_if_available( + &from); + if (source == NULL) { + ::google::protobuf::internal::ReflectionOps::Merge(from, this); + } else { + MergeFrom(*source); + } +} + +void NetworkProto::MergeFrom(const NetworkProto& from) { + GOOGLE_CHECK_NE(&from, this); + icodec_.MergeFrom(from.icodec_); + codec_.MergeFrom(from.codec_); + attribute_.MergeFrom(from.attribute_); + weights_.MergeFrom(from.weights_); + sub_.MergeFrom(from.sub_); + if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (from.has_kind()) { + set_kind(from.kind()); + } + if (from.has_name()) { + set_name(from.name()); + } + if (from.has_ninput()) { + set_ninput(from.ninput()); + } + if (from.has_noutput()) { + set_noutput(from.noutput()); + } + } + mutable_unknown_fields()->MergeFrom(from.unknown_fields()); +} + +void NetworkProto::CopyFrom(const ::google::protobuf::Message& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +void NetworkProto::CopyFrom(const NetworkProto& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool NetworkProto::IsInitialized() const { + if ((_has_bits_[0] & 0x0000000d) != 0x0000000d) return false; + + if (!::google::protobuf::internal::AllAreInitialized(this->attribute())) return false; + if (!::google::protobuf::internal::AllAreInitialized(this->sub())) return false; + return true; +} + +void NetworkProto::Swap(NetworkProto* other) { + if (other != this) { + std::swap(kind_, other->kind_); + std::swap(name_, other->name_); + std::swap(ninput_, other->ninput_); + std::swap(noutput_, other->noutput_); + icodec_.Swap(&other->icodec_); + codec_.Swap(&other->codec_); + attribute_.Swap(&other->attribute_); + weights_.Swap(&other->weights_); + sub_.Swap(&other->sub_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.Swap(&other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::google::protobuf::Metadata NetworkProto::GetMetadata() const { + protobuf_AssignDescriptorsOnce(); + ::google::protobuf::Metadata metadata; + metadata.descriptor = NetworkProto_descriptor_; + metadata.reflection = NetworkProto_reflection_; + return metadata; +} + + +// @@protoc_insertion_point(namespace_scope) + +} // namespace clstm + +// @@protoc_insertion_point(global_scope) diff --git a/clstm.pb.h b/clstm.pb.h new file mode 100644 index 0000000..9d7f627 --- /dev/null +++ b/clstm.pb.h @@ -0,0 +1,1102 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: clstm.proto + +#ifndef PROTOBUF_clstm_2eproto__INCLUDED +#define PROTOBUF_clstm_2eproto__INCLUDED + +#include + +#include + +#if GOOGLE_PROTOBUF_VERSION < 2006000 +#error This file was generated by a newer version of protoc which is +#error incompatible with your Protocol Buffer headers. Please update +#error your headers. +#endif +#if 2006001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION +#error This file was generated by an older version of protoc which is +#error incompatible with your Protocol Buffer headers. Please +#error regenerate this file with a newer version of protoc. +#endif + +#include +#include +#include +#include +#include +// @@protoc_insertion_point(includes) + +namespace clstm { + +// Internal implementation detail -- do not call these. +void protobuf_AddDesc_clstm_2eproto(); +void protobuf_AssignDesc_clstm_2eproto(); +void protobuf_ShutdownFile_clstm_2eproto(); + +class KeyValue; +class Array; +class NetworkProto; + +// =================================================================== + +class KeyValue : public ::google::protobuf::Message { + public: + KeyValue(); + virtual ~KeyValue(); + + KeyValue(const KeyValue& from); + + inline KeyValue& operator=(const KeyValue& from) { + CopyFrom(from); + return *this; + } + + inline const ::google::protobuf::UnknownFieldSet& unknown_fields() const { + return _unknown_fields_; + } + + inline ::google::protobuf::UnknownFieldSet* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const ::google::protobuf::Descriptor* descriptor(); + static const KeyValue& default_instance(); + + void Swap(KeyValue* other); + + // implements Message ---------------------------------------------- + + KeyValue* New() const; + void CopyFrom(const ::google::protobuf::Message& from); + void MergeFrom(const ::google::protobuf::Message& from); + void CopyFrom(const KeyValue& from); + void MergeFrom(const KeyValue& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + ::google::protobuf::uint8* SerializeWithCachedSizesToArray(::google::protobuf::uint8* output) const; + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::google::protobuf::Metadata GetMetadata() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // required string key = 1; + inline bool has_key() const; + inline void clear_key(); + static const int kKeyFieldNumber = 1; + inline const ::std::string& key() const; + inline void set_key(const ::std::string& value); + inline void set_key(const char* value); + inline void set_key(const char* value, size_t size); + inline ::std::string* mutable_key(); + inline ::std::string* release_key(); + inline void set_allocated_key(::std::string* key); + + // required string value = 2; + inline bool has_value() const; + inline void clear_value(); + static const int kValueFieldNumber = 2; + inline const ::std::string& value() const; + inline void set_value(const ::std::string& value); + inline void set_value(const char* value); + inline void set_value(const char* value, size_t size); + inline ::std::string* mutable_value(); + inline ::std::string* release_value(); + inline void set_allocated_value(::std::string* value); + + // @@protoc_insertion_point(class_scope:clstm.KeyValue) + private: + inline void set_has_key(); + inline void clear_has_key(); + inline void set_has_value(); + inline void clear_has_value(); + + ::google::protobuf::UnknownFieldSet _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::std::string* key_; + ::std::string* value_; + friend void protobuf_AddDesc_clstm_2eproto(); + friend void protobuf_AssignDesc_clstm_2eproto(); + friend void protobuf_ShutdownFile_clstm_2eproto(); + + void InitAsDefaultInstance(); + static KeyValue* default_instance_; +}; +// ------------------------------------------------------------------- + +class Array : public ::google::protobuf::Message { + public: + Array(); + virtual ~Array(); + + Array(const Array& from); + + inline Array& operator=(const Array& from) { + CopyFrom(from); + return *this; + } + + inline const ::google::protobuf::UnknownFieldSet& unknown_fields() const { + return _unknown_fields_; + } + + inline ::google::protobuf::UnknownFieldSet* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const ::google::protobuf::Descriptor* descriptor(); + static const Array& default_instance(); + + void Swap(Array* other); + + // implements Message ---------------------------------------------- + + Array* New() const; + void CopyFrom(const ::google::protobuf::Message& from); + void MergeFrom(const ::google::protobuf::Message& from); + void CopyFrom(const Array& from); + void MergeFrom(const Array& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + ::google::protobuf::uint8* SerializeWithCachedSizesToArray(::google::protobuf::uint8* output) const; + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::google::protobuf::Metadata GetMetadata() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // optional string name = 1; + inline bool has_name() const; + inline void clear_name(); + static const int kNameFieldNumber = 1; + inline const ::std::string& name() const; + inline void set_name(const ::std::string& value); + inline void set_name(const char* value); + inline void set_name(const char* value, size_t size); + inline ::std::string* mutable_name(); + inline ::std::string* release_name(); + inline void set_allocated_name(::std::string* name); + + // repeated int32 dim = 2; + inline int dim_size() const; + inline void clear_dim(); + static const int kDimFieldNumber = 2; + inline ::google::protobuf::int32 dim(int index) const; + inline void set_dim(int index, ::google::protobuf::int32 value); + inline void add_dim(::google::protobuf::int32 value); + inline const ::google::protobuf::RepeatedField< ::google::protobuf::int32 >& + dim() const; + inline ::google::protobuf::RepeatedField< ::google::protobuf::int32 >* + mutable_dim(); + + // repeated float value = 3; + inline int value_size() const; + inline void clear_value(); + static const int kValueFieldNumber = 3; + inline float value(int index) const; + inline void set_value(int index, float value); + inline void add_value(float value); + inline const ::google::protobuf::RepeatedField< float >& + value() const; + inline ::google::protobuf::RepeatedField< float >* + mutable_value(); + + // @@protoc_insertion_point(class_scope:clstm.Array) + private: + inline void set_has_name(); + inline void clear_has_name(); + + ::google::protobuf::UnknownFieldSet _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::std::string* name_; + ::google::protobuf::RepeatedField< ::google::protobuf::int32 > dim_; + ::google::protobuf::RepeatedField< float > value_; + friend void protobuf_AddDesc_clstm_2eproto(); + friend void protobuf_AssignDesc_clstm_2eproto(); + friend void protobuf_ShutdownFile_clstm_2eproto(); + + void InitAsDefaultInstance(); + static Array* default_instance_; +}; +// ------------------------------------------------------------------- + +class NetworkProto : public ::google::protobuf::Message { + public: + NetworkProto(); + virtual ~NetworkProto(); + + NetworkProto(const NetworkProto& from); + + inline NetworkProto& operator=(const NetworkProto& from) { + CopyFrom(from); + return *this; + } + + inline const ::google::protobuf::UnknownFieldSet& unknown_fields() const { + return _unknown_fields_; + } + + inline ::google::protobuf::UnknownFieldSet* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const ::google::protobuf::Descriptor* descriptor(); + static const NetworkProto& default_instance(); + + void Swap(NetworkProto* other); + + // implements Message ---------------------------------------------- + + NetworkProto* New() const; + void CopyFrom(const ::google::protobuf::Message& from); + void MergeFrom(const ::google::protobuf::Message& from); + void CopyFrom(const NetworkProto& from); + void MergeFrom(const NetworkProto& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + ::google::protobuf::uint8* SerializeWithCachedSizesToArray(::google::protobuf::uint8* output) const; + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::google::protobuf::Metadata GetMetadata() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // required string kind = 1; + inline bool has_kind() const; + inline void clear_kind(); + static const int kKindFieldNumber = 1; + inline const ::std::string& kind() const; + inline void set_kind(const ::std::string& value); + inline void set_kind(const char* value); + inline void set_kind(const char* value, size_t size); + inline ::std::string* mutable_kind(); + inline ::std::string* release_kind(); + inline void set_allocated_kind(::std::string* kind); + + // optional string name = 2; + inline bool has_name() const; + inline void clear_name(); + static const int kNameFieldNumber = 2; + inline const ::std::string& name() const; + inline void set_name(const ::std::string& value); + inline void set_name(const char* value); + inline void set_name(const char* value, size_t size); + inline ::std::string* mutable_name(); + inline ::std::string* release_name(); + inline void set_allocated_name(::std::string* name); + + // required int32 ninput = 10; + inline bool has_ninput() const; + inline void clear_ninput(); + static const int kNinputFieldNumber = 10; + inline ::google::protobuf::int32 ninput() const; + inline void set_ninput(::google::protobuf::int32 value); + + // required int32 noutput = 11; + inline bool has_noutput() const; + inline void clear_noutput(); + static const int kNoutputFieldNumber = 11; + inline ::google::protobuf::int32 noutput() const; + inline void set_noutput(::google::protobuf::int32 value); + + // repeated int32 icodec = 12; + inline int icodec_size() const; + inline void clear_icodec(); + static const int kIcodecFieldNumber = 12; + inline ::google::protobuf::int32 icodec(int index) const; + inline void set_icodec(int index, ::google::protobuf::int32 value); + inline void add_icodec(::google::protobuf::int32 value); + inline const ::google::protobuf::RepeatedField< ::google::protobuf::int32 >& + icodec() const; + inline ::google::protobuf::RepeatedField< ::google::protobuf::int32 >* + mutable_icodec(); + + // repeated int32 codec = 13; + inline int codec_size() const; + inline void clear_codec(); + static const int kCodecFieldNumber = 13; + inline ::google::protobuf::int32 codec(int index) const; + inline void set_codec(int index, ::google::protobuf::int32 value); + inline void add_codec(::google::protobuf::int32 value); + inline const ::google::protobuf::RepeatedField< ::google::protobuf::int32 >& + codec() const; + inline ::google::protobuf::RepeatedField< ::google::protobuf::int32 >* + mutable_codec(); + + // repeated .clstm.KeyValue attribute = 20; + inline int attribute_size() const; + inline void clear_attribute(); + static const int kAttributeFieldNumber = 20; + inline const ::clstm::KeyValue& attribute(int index) const; + inline ::clstm::KeyValue* mutable_attribute(int index); + inline ::clstm::KeyValue* add_attribute(); + inline const ::google::protobuf::RepeatedPtrField< ::clstm::KeyValue >& + attribute() const; + inline ::google::protobuf::RepeatedPtrField< ::clstm::KeyValue >* + mutable_attribute(); + + // repeated .clstm.Array weights = 30; + inline int weights_size() const; + inline void clear_weights(); + static const int kWeightsFieldNumber = 30; + inline const ::clstm::Array& weights(int index) const; + inline ::clstm::Array* mutable_weights(int index); + inline ::clstm::Array* add_weights(); + inline const ::google::protobuf::RepeatedPtrField< ::clstm::Array >& + weights() const; + inline ::google::protobuf::RepeatedPtrField< ::clstm::Array >* + mutable_weights(); + + // repeated .clstm.NetworkProto sub = 40; + inline int sub_size() const; + inline void clear_sub(); + static const int kSubFieldNumber = 40; + inline const ::clstm::NetworkProto& sub(int index) const; + inline ::clstm::NetworkProto* mutable_sub(int index); + inline ::clstm::NetworkProto* add_sub(); + inline const ::google::protobuf::RepeatedPtrField< ::clstm::NetworkProto >& + sub() const; + inline ::google::protobuf::RepeatedPtrField< ::clstm::NetworkProto >* + mutable_sub(); + + // @@protoc_insertion_point(class_scope:clstm.NetworkProto) + private: + inline void set_has_kind(); + inline void clear_has_kind(); + inline void set_has_name(); + inline void clear_has_name(); + inline void set_has_ninput(); + inline void clear_has_ninput(); + inline void set_has_noutput(); + inline void clear_has_noutput(); + + ::google::protobuf::UnknownFieldSet _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::std::string* kind_; + ::std::string* name_; + ::google::protobuf::int32 ninput_; + ::google::protobuf::int32 noutput_; + ::google::protobuf::RepeatedField< ::google::protobuf::int32 > icodec_; + ::google::protobuf::RepeatedField< ::google::protobuf::int32 > codec_; + ::google::protobuf::RepeatedPtrField< ::clstm::KeyValue > attribute_; + ::google::protobuf::RepeatedPtrField< ::clstm::Array > weights_; + ::google::protobuf::RepeatedPtrField< ::clstm::NetworkProto > sub_; + friend void protobuf_AddDesc_clstm_2eproto(); + friend void protobuf_AssignDesc_clstm_2eproto(); + friend void protobuf_ShutdownFile_clstm_2eproto(); + + void InitAsDefaultInstance(); + static NetworkProto* default_instance_; +}; +// =================================================================== + + +// =================================================================== + +// KeyValue + +// required string key = 1; +inline bool KeyValue::has_key() const { + return (_has_bits_[0] & 0x00000001u) != 0; +} +inline void KeyValue::set_has_key() { + _has_bits_[0] |= 0x00000001u; +} +inline void KeyValue::clear_has_key() { + _has_bits_[0] &= ~0x00000001u; +} +inline void KeyValue::clear_key() { + if (key_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + key_->clear(); + } + clear_has_key(); +} +inline const ::std::string& KeyValue::key() const { + // @@protoc_insertion_point(field_get:clstm.KeyValue.key) + return *key_; +} +inline void KeyValue::set_key(const ::std::string& value) { + set_has_key(); + if (key_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + key_ = new ::std::string; + } + key_->assign(value); + // @@protoc_insertion_point(field_set:clstm.KeyValue.key) +} +inline void KeyValue::set_key(const char* value) { + set_has_key(); + if (key_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + key_ = new ::std::string; + } + key_->assign(value); + // @@protoc_insertion_point(field_set_char:clstm.KeyValue.key) +} +inline void KeyValue::set_key(const char* value, size_t size) { + set_has_key(); + if (key_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + key_ = new ::std::string; + } + key_->assign(reinterpret_cast(value), size); + // @@protoc_insertion_point(field_set_pointer:clstm.KeyValue.key) +} +inline ::std::string* KeyValue::mutable_key() { + set_has_key(); + if (key_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + key_ = new ::std::string; + } + // @@protoc_insertion_point(field_mutable:clstm.KeyValue.key) + return key_; +} +inline ::std::string* KeyValue::release_key() { + clear_has_key(); + if (key_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + return NULL; + } else { + ::std::string* temp = key_; + key_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return temp; + } +} +inline void KeyValue::set_allocated_key(::std::string* key) { + if (key_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete key_; + } + if (key) { + set_has_key(); + key_ = key; + } else { + clear_has_key(); + key_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + } + // @@protoc_insertion_point(field_set_allocated:clstm.KeyValue.key) +} + +// required string value = 2; +inline bool KeyValue::has_value() const { + return (_has_bits_[0] & 0x00000002u) != 0; +} +inline void KeyValue::set_has_value() { + _has_bits_[0] |= 0x00000002u; +} +inline void KeyValue::clear_has_value() { + _has_bits_[0] &= ~0x00000002u; +} +inline void KeyValue::clear_value() { + if (value_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + value_->clear(); + } + clear_has_value(); +} +inline const ::std::string& KeyValue::value() const { + // @@protoc_insertion_point(field_get:clstm.KeyValue.value) + return *value_; +} +inline void KeyValue::set_value(const ::std::string& value) { + set_has_value(); + if (value_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + value_ = new ::std::string; + } + value_->assign(value); + // @@protoc_insertion_point(field_set:clstm.KeyValue.value) +} +inline void KeyValue::set_value(const char* value) { + set_has_value(); + if (value_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + value_ = new ::std::string; + } + value_->assign(value); + // @@protoc_insertion_point(field_set_char:clstm.KeyValue.value) +} +inline void KeyValue::set_value(const char* value, size_t size) { + set_has_value(); + if (value_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + value_ = new ::std::string; + } + value_->assign(reinterpret_cast(value), size); + // @@protoc_insertion_point(field_set_pointer:clstm.KeyValue.value) +} +inline ::std::string* KeyValue::mutable_value() { + set_has_value(); + if (value_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + value_ = new ::std::string; + } + // @@protoc_insertion_point(field_mutable:clstm.KeyValue.value) + return value_; +} +inline ::std::string* KeyValue::release_value() { + clear_has_value(); + if (value_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + return NULL; + } else { + ::std::string* temp = value_; + value_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return temp; + } +} +inline void KeyValue::set_allocated_value(::std::string* value) { + if (value_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete value_; + } + if (value) { + set_has_value(); + value_ = value; + } else { + clear_has_value(); + value_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + } + // @@protoc_insertion_point(field_set_allocated:clstm.KeyValue.value) +} + +// ------------------------------------------------------------------- + +// Array + +// optional string name = 1; +inline bool Array::has_name() const { + return (_has_bits_[0] & 0x00000001u) != 0; +} +inline void Array::set_has_name() { + _has_bits_[0] |= 0x00000001u; +} +inline void Array::clear_has_name() { + _has_bits_[0] &= ~0x00000001u; +} +inline void Array::clear_name() { + if (name_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + name_->clear(); + } + clear_has_name(); +} +inline const ::std::string& Array::name() const { + // @@protoc_insertion_point(field_get:clstm.Array.name) + return *name_; +} +inline void Array::set_name(const ::std::string& value) { + set_has_name(); + if (name_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + name_ = new ::std::string; + } + name_->assign(value); + // @@protoc_insertion_point(field_set:clstm.Array.name) +} +inline void Array::set_name(const char* value) { + set_has_name(); + if (name_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + name_ = new ::std::string; + } + name_->assign(value); + // @@protoc_insertion_point(field_set_char:clstm.Array.name) +} +inline void Array::set_name(const char* value, size_t size) { + set_has_name(); + if (name_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + name_ = new ::std::string; + } + name_->assign(reinterpret_cast(value), size); + // @@protoc_insertion_point(field_set_pointer:clstm.Array.name) +} +inline ::std::string* Array::mutable_name() { + set_has_name(); + if (name_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + name_ = new ::std::string; + } + // @@protoc_insertion_point(field_mutable:clstm.Array.name) + return name_; +} +inline ::std::string* Array::release_name() { + clear_has_name(); + if (name_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + return NULL; + } else { + ::std::string* temp = name_; + name_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return temp; + } +} +inline void Array::set_allocated_name(::std::string* name) { + if (name_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete name_; + } + if (name) { + set_has_name(); + name_ = name; + } else { + clear_has_name(); + name_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + } + // @@protoc_insertion_point(field_set_allocated:clstm.Array.name) +} + +// repeated int32 dim = 2; +inline int Array::dim_size() const { + return dim_.size(); +} +inline void Array::clear_dim() { + dim_.Clear(); +} +inline ::google::protobuf::int32 Array::dim(int index) const { + // @@protoc_insertion_point(field_get:clstm.Array.dim) + return dim_.Get(index); +} +inline void Array::set_dim(int index, ::google::protobuf::int32 value) { + dim_.Set(index, value); + // @@protoc_insertion_point(field_set:clstm.Array.dim) +} +inline void Array::add_dim(::google::protobuf::int32 value) { + dim_.Add(value); + // @@protoc_insertion_point(field_add:clstm.Array.dim) +} +inline const ::google::protobuf::RepeatedField< ::google::protobuf::int32 >& +Array::dim() const { + // @@protoc_insertion_point(field_list:clstm.Array.dim) + return dim_; +} +inline ::google::protobuf::RepeatedField< ::google::protobuf::int32 >* +Array::mutable_dim() { + // @@protoc_insertion_point(field_mutable_list:clstm.Array.dim) + return &dim_; +} + +// repeated float value = 3; +inline int Array::value_size() const { + return value_.size(); +} +inline void Array::clear_value() { + value_.Clear(); +} +inline float Array::value(int index) const { + // @@protoc_insertion_point(field_get:clstm.Array.value) + return value_.Get(index); +} +inline void Array::set_value(int index, float value) { + value_.Set(index, value); + // @@protoc_insertion_point(field_set:clstm.Array.value) +} +inline void Array::add_value(float value) { + value_.Add(value); + // @@protoc_insertion_point(field_add:clstm.Array.value) +} +inline const ::google::protobuf::RepeatedField< float >& +Array::value() const { + // @@protoc_insertion_point(field_list:clstm.Array.value) + return value_; +} +inline ::google::protobuf::RepeatedField< float >* +Array::mutable_value() { + // @@protoc_insertion_point(field_mutable_list:clstm.Array.value) + return &value_; +} + +// ------------------------------------------------------------------- + +// NetworkProto + +// required string kind = 1; +inline bool NetworkProto::has_kind() const { + return (_has_bits_[0] & 0x00000001u) != 0; +} +inline void NetworkProto::set_has_kind() { + _has_bits_[0] |= 0x00000001u; +} +inline void NetworkProto::clear_has_kind() { + _has_bits_[0] &= ~0x00000001u; +} +inline void NetworkProto::clear_kind() { + if (kind_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + kind_->clear(); + } + clear_has_kind(); +} +inline const ::std::string& NetworkProto::kind() const { + // @@protoc_insertion_point(field_get:clstm.NetworkProto.kind) + return *kind_; +} +inline void NetworkProto::set_kind(const ::std::string& value) { + set_has_kind(); + if (kind_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + kind_ = new ::std::string; + } + kind_->assign(value); + // @@protoc_insertion_point(field_set:clstm.NetworkProto.kind) +} +inline void NetworkProto::set_kind(const char* value) { + set_has_kind(); + if (kind_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + kind_ = new ::std::string; + } + kind_->assign(value); + // @@protoc_insertion_point(field_set_char:clstm.NetworkProto.kind) +} +inline void NetworkProto::set_kind(const char* value, size_t size) { + set_has_kind(); + if (kind_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + kind_ = new ::std::string; + } + kind_->assign(reinterpret_cast(value), size); + // @@protoc_insertion_point(field_set_pointer:clstm.NetworkProto.kind) +} +inline ::std::string* NetworkProto::mutable_kind() { + set_has_kind(); + if (kind_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + kind_ = new ::std::string; + } + // @@protoc_insertion_point(field_mutable:clstm.NetworkProto.kind) + return kind_; +} +inline ::std::string* NetworkProto::release_kind() { + clear_has_kind(); + if (kind_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + return NULL; + } else { + ::std::string* temp = kind_; + kind_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return temp; + } +} +inline void NetworkProto::set_allocated_kind(::std::string* kind) { + if (kind_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete kind_; + } + if (kind) { + set_has_kind(); + kind_ = kind; + } else { + clear_has_kind(); + kind_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + } + // @@protoc_insertion_point(field_set_allocated:clstm.NetworkProto.kind) +} + +// optional string name = 2; +inline bool NetworkProto::has_name() const { + return (_has_bits_[0] & 0x00000002u) != 0; +} +inline void NetworkProto::set_has_name() { + _has_bits_[0] |= 0x00000002u; +} +inline void NetworkProto::clear_has_name() { + _has_bits_[0] &= ~0x00000002u; +} +inline void NetworkProto::clear_name() { + if (name_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + name_->clear(); + } + clear_has_name(); +} +inline const ::std::string& NetworkProto::name() const { + // @@protoc_insertion_point(field_get:clstm.NetworkProto.name) + return *name_; +} +inline void NetworkProto::set_name(const ::std::string& value) { + set_has_name(); + if (name_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + name_ = new ::std::string; + } + name_->assign(value); + // @@protoc_insertion_point(field_set:clstm.NetworkProto.name) +} +inline void NetworkProto::set_name(const char* value) { + set_has_name(); + if (name_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + name_ = new ::std::string; + } + name_->assign(value); + // @@protoc_insertion_point(field_set_char:clstm.NetworkProto.name) +} +inline void NetworkProto::set_name(const char* value, size_t size) { + set_has_name(); + if (name_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + name_ = new ::std::string; + } + name_->assign(reinterpret_cast(value), size); + // @@protoc_insertion_point(field_set_pointer:clstm.NetworkProto.name) +} +inline ::std::string* NetworkProto::mutable_name() { + set_has_name(); + if (name_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + name_ = new ::std::string; + } + // @@protoc_insertion_point(field_mutable:clstm.NetworkProto.name) + return name_; +} +inline ::std::string* NetworkProto::release_name() { + clear_has_name(); + if (name_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + return NULL; + } else { + ::std::string* temp = name_; + name_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return temp; + } +} +inline void NetworkProto::set_allocated_name(::std::string* name) { + if (name_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete name_; + } + if (name) { + set_has_name(); + name_ = name; + } else { + clear_has_name(); + name_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + } + // @@protoc_insertion_point(field_set_allocated:clstm.NetworkProto.name) +} + +// required int32 ninput = 10; +inline bool NetworkProto::has_ninput() const { + return (_has_bits_[0] & 0x00000004u) != 0; +} +inline void NetworkProto::set_has_ninput() { + _has_bits_[0] |= 0x00000004u; +} +inline void NetworkProto::clear_has_ninput() { + _has_bits_[0] &= ~0x00000004u; +} +inline void NetworkProto::clear_ninput() { + ninput_ = 0; + clear_has_ninput(); +} +inline ::google::protobuf::int32 NetworkProto::ninput() const { + // @@protoc_insertion_point(field_get:clstm.NetworkProto.ninput) + return ninput_; +} +inline void NetworkProto::set_ninput(::google::protobuf::int32 value) { + set_has_ninput(); + ninput_ = value; + // @@protoc_insertion_point(field_set:clstm.NetworkProto.ninput) +} + +// required int32 noutput = 11; +inline bool NetworkProto::has_noutput() const { + return (_has_bits_[0] & 0x00000008u) != 0; +} +inline void NetworkProto::set_has_noutput() { + _has_bits_[0] |= 0x00000008u; +} +inline void NetworkProto::clear_has_noutput() { + _has_bits_[0] &= ~0x00000008u; +} +inline void NetworkProto::clear_noutput() { + noutput_ = 0; + clear_has_noutput(); +} +inline ::google::protobuf::int32 NetworkProto::noutput() const { + // @@protoc_insertion_point(field_get:clstm.NetworkProto.noutput) + return noutput_; +} +inline void NetworkProto::set_noutput(::google::protobuf::int32 value) { + set_has_noutput(); + noutput_ = value; + // @@protoc_insertion_point(field_set:clstm.NetworkProto.noutput) +} + +// repeated int32 icodec = 12; +inline int NetworkProto::icodec_size() const { + return icodec_.size(); +} +inline void NetworkProto::clear_icodec() { + icodec_.Clear(); +} +inline ::google::protobuf::int32 NetworkProto::icodec(int index) const { + // @@protoc_insertion_point(field_get:clstm.NetworkProto.icodec) + return icodec_.Get(index); +} +inline void NetworkProto::set_icodec(int index, ::google::protobuf::int32 value) { + icodec_.Set(index, value); + // @@protoc_insertion_point(field_set:clstm.NetworkProto.icodec) +} +inline void NetworkProto::add_icodec(::google::protobuf::int32 value) { + icodec_.Add(value); + // @@protoc_insertion_point(field_add:clstm.NetworkProto.icodec) +} +inline const ::google::protobuf::RepeatedField< ::google::protobuf::int32 >& +NetworkProto::icodec() const { + // @@protoc_insertion_point(field_list:clstm.NetworkProto.icodec) + return icodec_; +} +inline ::google::protobuf::RepeatedField< ::google::protobuf::int32 >* +NetworkProto::mutable_icodec() { + // @@protoc_insertion_point(field_mutable_list:clstm.NetworkProto.icodec) + return &icodec_; +} + +// repeated int32 codec = 13; +inline int NetworkProto::codec_size() const { + return codec_.size(); +} +inline void NetworkProto::clear_codec() { + codec_.Clear(); +} +inline ::google::protobuf::int32 NetworkProto::codec(int index) const { + // @@protoc_insertion_point(field_get:clstm.NetworkProto.codec) + return codec_.Get(index); +} +inline void NetworkProto::set_codec(int index, ::google::protobuf::int32 value) { + codec_.Set(index, value); + // @@protoc_insertion_point(field_set:clstm.NetworkProto.codec) +} +inline void NetworkProto::add_codec(::google::protobuf::int32 value) { + codec_.Add(value); + // @@protoc_insertion_point(field_add:clstm.NetworkProto.codec) +} +inline const ::google::protobuf::RepeatedField< ::google::protobuf::int32 >& +NetworkProto::codec() const { + // @@protoc_insertion_point(field_list:clstm.NetworkProto.codec) + return codec_; +} +inline ::google::protobuf::RepeatedField< ::google::protobuf::int32 >* +NetworkProto::mutable_codec() { + // @@protoc_insertion_point(field_mutable_list:clstm.NetworkProto.codec) + return &codec_; +} + +// repeated .clstm.KeyValue attribute = 20; +inline int NetworkProto::attribute_size() const { + return attribute_.size(); +} +inline void NetworkProto::clear_attribute() { + attribute_.Clear(); +} +inline const ::clstm::KeyValue& NetworkProto::attribute(int index) const { + // @@protoc_insertion_point(field_get:clstm.NetworkProto.attribute) + return attribute_.Get(index); +} +inline ::clstm::KeyValue* NetworkProto::mutable_attribute(int index) { + // @@protoc_insertion_point(field_mutable:clstm.NetworkProto.attribute) + return attribute_.Mutable(index); +} +inline ::clstm::KeyValue* NetworkProto::add_attribute() { + // @@protoc_insertion_point(field_add:clstm.NetworkProto.attribute) + return attribute_.Add(); +} +inline const ::google::protobuf::RepeatedPtrField< ::clstm::KeyValue >& +NetworkProto::attribute() const { + // @@protoc_insertion_point(field_list:clstm.NetworkProto.attribute) + return attribute_; +} +inline ::google::protobuf::RepeatedPtrField< ::clstm::KeyValue >* +NetworkProto::mutable_attribute() { + // @@protoc_insertion_point(field_mutable_list:clstm.NetworkProto.attribute) + return &attribute_; +} + +// repeated .clstm.Array weights = 30; +inline int NetworkProto::weights_size() const { + return weights_.size(); +} +inline void NetworkProto::clear_weights() { + weights_.Clear(); +} +inline const ::clstm::Array& NetworkProto::weights(int index) const { + // @@protoc_insertion_point(field_get:clstm.NetworkProto.weights) + return weights_.Get(index); +} +inline ::clstm::Array* NetworkProto::mutable_weights(int index) { + // @@protoc_insertion_point(field_mutable:clstm.NetworkProto.weights) + return weights_.Mutable(index); +} +inline ::clstm::Array* NetworkProto::add_weights() { + // @@protoc_insertion_point(field_add:clstm.NetworkProto.weights) + return weights_.Add(); +} +inline const ::google::protobuf::RepeatedPtrField< ::clstm::Array >& +NetworkProto::weights() const { + // @@protoc_insertion_point(field_list:clstm.NetworkProto.weights) + return weights_; +} +inline ::google::protobuf::RepeatedPtrField< ::clstm::Array >* +NetworkProto::mutable_weights() { + // @@protoc_insertion_point(field_mutable_list:clstm.NetworkProto.weights) + return &weights_; +} + +// repeated .clstm.NetworkProto sub = 40; +inline int NetworkProto::sub_size() const { + return sub_.size(); +} +inline void NetworkProto::clear_sub() { + sub_.Clear(); +} +inline const ::clstm::NetworkProto& NetworkProto::sub(int index) const { + // @@protoc_insertion_point(field_get:clstm.NetworkProto.sub) + return sub_.Get(index); +} +inline ::clstm::NetworkProto* NetworkProto::mutable_sub(int index) { + // @@protoc_insertion_point(field_mutable:clstm.NetworkProto.sub) + return sub_.Mutable(index); +} +inline ::clstm::NetworkProto* NetworkProto::add_sub() { + // @@protoc_insertion_point(field_add:clstm.NetworkProto.sub) + return sub_.Add(); +} +inline const ::google::protobuf::RepeatedPtrField< ::clstm::NetworkProto >& +NetworkProto::sub() const { + // @@protoc_insertion_point(field_list:clstm.NetworkProto.sub) + return sub_; +} +inline ::google::protobuf::RepeatedPtrField< ::clstm::NetworkProto >* +NetworkProto::mutable_sub() { + // @@protoc_insertion_point(field_mutable_list:clstm.NetworkProto.sub) + return &sub_; +} + + +// @@protoc_insertion_point(namespace_scope) + +} // namespace clstm + +#ifndef SWIG +namespace google { +namespace protobuf { + + +} // namespace google +} // namespace protobuf +#endif // SWIG + +// @@protoc_insertion_point(global_scope) + +#endif // PROTOBUF_clstm_2eproto__INCLUDED From 56ca5668d345d5f94a4e0d96cc2eca6e593b33a2 Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Sun, 14 Aug 2016 18:04:58 +0200 Subject: [PATCH 11/34] Fix bug in protobuf stale check --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1211308..599dade 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,8 @@ def ensure_protobuf(): exists = os.path.exists("clstm.pb.cc") - stale = os.path.getctime("clstm.pb.cc") < os.path.getctime("clstm.proto") + stale = (exists and + os.path.getctime("clstm.pb.cc") < os.path.getctime("clstm.proto")) if not exists or stale: print "Generating proto file" os.system("protoc clstm.proto --cpp_out=.") From 4e2a7aa813f9256b29fe173a02009112545fa6b6 Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Sun, 14 Aug 2016 18:05:22 +0200 Subject: [PATCH 12/34] python: allow loading of model in constructor --- pyclstm.pyx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyclstm.pyx b/pyclstm.pyx index 2c5436b..347dbb8 100644 --- a/pyclstm.pyx +++ b/pyclstm.pyx @@ -52,6 +52,8 @@ cdef class ClstmOcr: def __cinit__(self, str fname=None): self._ocr = new _clstm.CLSTMOCR() + if fname: + self.load(fname) cpdef load(self, str fname): cdef bint rv = self._ocr.maybe_load(fname) From d17cea10e4917aba8d5fc528065125177de38836 Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Sun, 14 Aug 2016 18:55:22 +0200 Subject: [PATCH 13/34] python: Add option to use numpy array as image data --- pyclstm.pyx | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/pyclstm.pyx b/pyclstm.pyx index 347dbb8..cebba4d 100644 --- a/pyclstm.pyx +++ b/pyclstm.pyx @@ -47,6 +47,21 @@ cdef load_img(img, _clstm.Tensor2 *data): data[0].put(px, i, j) +cdef load_nparray(npdata, _clstm.Tensor2 *data): + """ Copy image data from a numpy array to an Eigen tensor. + + This assumes that all pixels are floating point numbers in the range [0, 1] + where 0 is white and 1 is black. + + Currently this copies all data, but it should be possible to make Eigen + use the numpy-allocated data. + """ + data.resize(npdata.shape[0], npdata.shape[1]) + for i in range(npdata.shape[0]): + for j in range(npdata.shape[1]): + data[0].put(npdata[i,j], i, j) + + cdef class ClstmOcr: cdef _clstm.CLSTMOCR *_ocr @@ -85,13 +100,19 @@ cdef class ClstmOcr: def train(self, img, unicode text): cdef _clstm.Tensor2 data - load_img(img, &data) + if hasattr(img, 'width'): + load_img(img, &data) + elif hasattr(img, 'shape'): + load_nparray(img, &data) return self._ocr.train_utf8( data.map(), text.encode('utf8')).decode('utf8') def recognize(self, img): cdef _clstm.Tensor2 data - load_img(img, &data) + if hasattr(img, 'width'): + load_img(img, &data) + elif hasattr(img, 'shape'): + load_nparray(img, &data) return self._ocr.predict_utf8(data.map()).decode('utf8') def recognize_chars(self, img): @@ -99,7 +120,10 @@ cdef class ClstmOcr: cdef vector[_clstm.CharPrediction] preds cdef vector[_clstm.CharPrediction].iterator pred_it cdef wchar_t[2] cur_char - load_img(img, &data) + if hasattr(img, 'width'): + load_img(img, &data) + elif hasattr(img, 'shape'): + load_nparray(img, &data) self._ocr.predict(preds, data.map()) for i in range(preds.size()): cur_char[0] = preds[i].c From 368e6af41327980e8e01c4507529fa2c221cc71d Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Thu, 6 Oct 2016 11:00:03 +0200 Subject: [PATCH 14/34] Combine create_bidi/set_learning_rate into a higher-level `prepare_training` method --- pyclstm.pyx | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pyclstm.pyx b/pyclstm.pyx index cebba4d..367ae4a 100644 --- a/pyclstm.pyx +++ b/pyclstm.pyx @@ -80,19 +80,18 @@ cdef class ClstmOcr: if not rv: raise IOError("Could not save model to {}".format(fname)) - cpdef create_bidi(self, chars, int num_hidden): - chars_str = u"".join(sorted(chars)) + cpdef prepare_training(self, lexicon, int num_hidden=100, + float learning_rate=0.0001, float momentum=0.9): + lexicon_str = u"".join(sorted(lexicon)) cdef vector[int] codec - cdef Py_ssize_t length = len(chars_str.encode("UTF-16")) // 2 + cdef Py_ssize_t length = len(lexicon_str.encode("UTF-16")) // 2 cdef wchar_t *wchars = malloc(length * sizeof(wchar_t)) cdef Py_ssize_t number_written = PyUnicode_AsWideChar( - chars_str, wchars, length) + lexicon_str, wchars, length) codec.push_back(0) for i in range(length-1): codec.push_back((wchars[i])) self._ocr.createBidi(codec, num_hidden) - - cpdef set_learning_rate(self, float learning_rate, float momentum): self._ocr.setLearningRate(learning_rate, momentum) def aligned(self): From 5e02b0fa96f19c04043de6c7426cf9148c6986a4 Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Thu, 6 Oct 2016 11:43:48 +0200 Subject: [PATCH 15/34] More docstrings --- pyclstm.pyx | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/pyclstm.pyx b/pyclstm.pyx index 367ae4a..8a636d8 100644 --- a/pyclstm.pyx +++ b/pyclstm.pyx @@ -22,6 +22,12 @@ cdef extern from "Python.h": cpdef double levenshtein(unicode a, unicode b): + """ Determine the Levenshtein-distance between to unicode strings. + + :type a: unicode + :type b: unicode + :rtype: int + """ return _clstm.levenshtein[string, string]( a.encode('utf8'), b.encode('utf8')) @@ -63,25 +69,73 @@ cdef load_nparray(npdata, _clstm.Tensor2 *data): cdef class ClstmOcr: + """ An OCR engine based on CLSTM, operating on line images. + + Use this class to either train your own OCR model or to load a + pre-trained model from disk. + + For training, set your parameters with :py:meth:`prepare_training`, and + then iteratively supply a line image (:py:class:`PIL.Image` or + :py:class:`numpy ndarray`) and the ground truth for the line to + :py:meth:`train`. Once finished with training, call :py:meth:`save` + to persist the trained model to disk. + + For prediction, two methods are available. The simplest, + :py:meth:`recognize` takes a line image (see above) and returns the + recognized text as a string. If more information about the recognized + text is needed, use :py:meth:`recognize-chars`, which returns a generator + that yields :py:class:`CharPrediction` objects that contain information + about each character (x-offset, confidence and recognized character). + """ cdef _clstm.CLSTMOCR *_ocr def __cinit__(self, str fname=None): + """ Initialize the OCR engine, optionally loading a model from disk. + + :param fname: Path to pre-trained model on disk + :type fname: str + """ self._ocr = new _clstm.CLSTMOCR() if fname: self.load(fname) cpdef load(self, str fname): + """ Load a pre-trained model from disk. + + :param fname: Path to pre-trained model on disk + :type fname: str + """ cdef bint rv = self._ocr.maybe_load(fname) if not rv: raise IOError("Could not load model from {}".format(fname)) cpdef save(self, str fname): + """ Save the model to disk. + + :param fname: Path to store model in + :type fname: str + """ cdef bint rv = self._ocr.maybe_save(fname) if not rv: raise IOError("Could not save model to {}".format(fname)) cpdef prepare_training(self, lexicon, int num_hidden=100, float learning_rate=0.0001, float momentum=0.9): + """ Prepare training by setting the lexicon and hyperparameters. + + :param lexicon: Iterable of characters that are to be recognized + by the OCR model, must not have duplicates + :type lexicon: iterable of str/unicode + :param num_hidden: Number of hidden units in the LSTM layers, larger + values require more storage/memory and take longer + for training and recognition, so try to find + a good performance/cost tradeoff. + :type num_hidden: int + :param learning_rate: Learning rate for the model training + :type learning_rate: float + :param momentum: Momentum for the model training + :type momentum: float + """ lexicon_str = u"".join(sorted(lexicon)) cdef vector[int] codec cdef Py_ssize_t length = len(lexicon_str.encode("UTF-16")) // 2 @@ -95,9 +149,24 @@ cdef class ClstmOcr: self._ocr.setLearningRate(learning_rate, momentum) def aligned(self): + """ Get the aligned output of the last trained sample. + + :rtype: unicode + """ return self._ocr.aligned_utf8().decode('utf8') def train(self, img, unicode text): + """ Train the model with a line image and its ground truth. + + :param img: The line image for the ground truth + :type img: :py:class:`PIL.Image`/:py:class:`numpy.ndarray` + :param text: The ground truth text for the line image + :type text: unicode + :returns: The recognized text for the line image, can be used + to estimate error against the ground truth + (via :py:function:`levenshtein`) + :rtype: unicode + """ cdef _clstm.Tensor2 data if hasattr(img, 'width'): load_img(img, &data) @@ -107,6 +176,13 @@ cdef class ClstmOcr: data.map(), text.encode('utf8')).decode('utf8') def recognize(self, img): + """ Recognize the text on the line image. + + :param img: The line image for the ground truth + :type img: :py:class:`PIL.Image`/:py:class:`numpy.ndarray` + :returns: The recognized text for the line + :rtype: unicode + """ cdef _clstm.Tensor2 data if hasattr(img, 'width'): load_img(img, &data) @@ -115,6 +191,15 @@ cdef class ClstmOcr: return self._ocr.predict_utf8(data.map()).decode('utf8') def recognize_chars(self, img): + """ Recognize the characters on the line, along with their position + and confidence. + + :param img: The line image for the ground truth + :type img: :py:class:`PIL.Image`/:py:class:`numpy.ndarray` + :returns: The recognized text for the line, represented as + information about its composing characters. + :rtype: generator that yield :py:class:`CharPrediction` + """ cdef _clstm.Tensor2 data cdef vector[_clstm.CharPrediction] preds cdef vector[_clstm.CharPrediction].iterator pred_it From 1b664434108db82d6dd97e89e21a6d40bb5af631 Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Thu, 6 Oct 2016 12:56:05 +0200 Subject: [PATCH 16/34] Embed function signatures into Python extension --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 599dade..2b0bbcc 100644 --- a/setup.py +++ b/setup.py @@ -35,4 +35,4 @@ def ensure_protobuf(): version='0.1', author="Thomas Breuel, Johannes Baiter", description="CLSTM Python bindings", - ext_modules=cythonize([ext])) + ext_modules=cythonize([ext], compiler_directives={'embedsignature': True})) From 7b32e65fa4f9c22f17ce2687ff1756d94833cf20 Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Thu, 6 Oct 2016 13:27:15 +0200 Subject: [PATCH 17/34] Add docs for Python extension --- .gitignore | 1 + pyclstm.pyx | 14 +- pydoc/Makefile | 225 +++++++++++++++++++++++++++++++ pydoc/conf.py | 347 ++++++++++++++++++++++++++++++++++++++++++++++++ pydoc/index.rst | 28 ++++ pydoc/make.bat | 281 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 889 insertions(+), 7 deletions(-) create mode 100644 pydoc/Makefile create mode 100644 pydoc/conf.py create mode 100644 pydoc/index.rst create mode 100644 pydoc/make.bat diff --git a/.gitignore b/.gitignore index a0be001..169b41d 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ build/ *.os *.a *.so +pydoc/_build diff --git a/pyclstm.pyx b/pyclstm.pyx index 8a636d8..6589605 100644 --- a/pyclstm.pyx +++ b/pyclstm.pyx @@ -75,15 +75,15 @@ cdef class ClstmOcr: pre-trained model from disk. For training, set your parameters with :py:meth:`prepare_training`, and - then iteratively supply a line image (:py:class:`PIL.Image` or - :py:class:`numpy ndarray`) and the ground truth for the line to + then iteratively supply a line image (:py:class:`PIL.Image.Image` or + :py:class:`numpy.ndarray`) and the ground truth for the line to :py:meth:`train`. Once finished with training, call :py:meth:`save` to persist the trained model to disk. For prediction, two methods are available. The simplest, :py:meth:`recognize` takes a line image (see above) and returns the recognized text as a string. If more information about the recognized - text is needed, use :py:meth:`recognize-chars`, which returns a generator + text is needed, use :py:meth:`recognize_chars`, which returns a generator that yields :py:class:`CharPrediction` objects that contain information about each character (x-offset, confidence and recognized character). """ @@ -159,12 +159,12 @@ cdef class ClstmOcr: """ Train the model with a line image and its ground truth. :param img: The line image for the ground truth - :type img: :py:class:`PIL.Image`/:py:class:`numpy.ndarray` + :type img: :py:class:`PIL.Image.Image`/:py:class:`numpy.ndarray` :param text: The ground truth text for the line image :type text: unicode :returns: The recognized text for the line image, can be used to estimate error against the ground truth - (via :py:function:`levenshtein`) + (via :py:func:`levenshtein`) :rtype: unicode """ cdef _clstm.Tensor2 data @@ -179,7 +179,7 @@ cdef class ClstmOcr: """ Recognize the text on the line image. :param img: The line image for the ground truth - :type img: :py:class:`PIL.Image`/:py:class:`numpy.ndarray` + :type img: :py:class:`PIL.Image.Image`/:py:class:`numpy.ndarray` :returns: The recognized text for the line :rtype: unicode """ @@ -195,7 +195,7 @@ cdef class ClstmOcr: and confidence. :param img: The line image for the ground truth - :type img: :py:class:`PIL.Image`/:py:class:`numpy.ndarray` + :type img: :py:class:`PIL.Image.Image`/:py:class:`numpy.ndarray` :returns: The recognized text for the line, represented as information about its composing characters. :rtype: generator that yield :py:class:`CharPrediction` diff --git a/pydoc/Makefile b/pydoc/Makefile new file mode 100644 index 0000000..4f4eedf --- /dev/null +++ b/pydoc/Makefile @@ -0,0 +1,225 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " epub3 to make an epub3" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + @echo " dummy to check syntax errors of document sources" + +.PHONY: clean +clean: + rm -rf $(BUILDDIR)/* + +.PHONY: html +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +.PHONY: dirhtml +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +.PHONY: singlehtml +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +.PHONY: pickle +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +.PHONY: json +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +.PHONY: htmlhelp +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +.PHONY: qthelp +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pyclstm.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pyclstm.qhc" + +.PHONY: applehelp +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +.PHONY: devhelp +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/pyclstm" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pyclstm" + @echo "# devhelp" + +.PHONY: epub +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +.PHONY: epub3 +epub3: + $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 + @echo + @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." + +.PHONY: latex +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +.PHONY: latexpdf +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: latexpdfja +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: text +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +.PHONY: man +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +.PHONY: texinfo +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +.PHONY: info +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +.PHONY: gettext +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +.PHONY: changes +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +.PHONY: linkcheck +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +.PHONY: doctest +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +.PHONY: coverage +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +.PHONY: xml +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +.PHONY: pseudoxml +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." + +.PHONY: dummy +dummy: + $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy + @echo + @echo "Build finished. Dummy builder generates no files." diff --git a/pydoc/conf.py b/pydoc/conf.py new file mode 100644 index 0000000..f225876 --- /dev/null +++ b/pydoc/conf.py @@ -0,0 +1,347 @@ +# -*- coding: utf-8 -*- +# +# pyclstm documentation build configuration file, created by +# sphinx-quickstart on Thu Oct 6 12:57:42 2016. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +# +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'pyclstm' +copyright = u'2016, Thomas Breuel, Johannes Baiter' +author = u'Thomas Breuel, Johannes Baiter' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'0.1' +# The full version, including alpha/beta/rc tags. +release = u'0.1' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# +# today = '' +# +# Else, today_fmt is used as the format for a strftime call. +# +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. +# " v documentation" by default. +# +# html_title = u'pyclstm v0.1' + +# A shorter title for the navigation bar. Default is the same as html_title. +# +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# +# html_logo = None + +# The name of an image file (relative to this directory) to use as a favicon of +# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# +# html_extra_path = [] + +# If not None, a 'Last updated on:' timestamp is inserted at every page +# bottom, using the given strftime format. +# The empty string is equivalent to '%b %d, %Y'. +# +# html_last_updated_fmt = None + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# +# html_additional_pages = {} + +# If false, no module index is generated. +# +# html_domain_indices = True + +# If false, no index is generated. +# +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh' +# +# html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# 'ja' uses this config value. +# 'zh' user can custom change `jieba` dictionary path. +# +# html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +# +# html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'pyclstmdoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'pyclstm.tex', u'pyclstm Documentation', + u'Thomas Breuel, Johannes Baiter', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# +# latex_use_parts = False + +# If true, show page references after internal links. +# +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# +# latex_appendices = [] + +# It false, will not define \strong, \code, itleref, \crossref ... but only +# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added +# packages. +# +# latex_keep_old_macro_names = True + +# If false, no module index is generated. +# +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'pyclstm', u'pyclstm Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +# +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'pyclstm', u'pyclstm Documentation', + author, 'pyclstm', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +# +# texinfo_appendices = [] + +# If false, no module index is generated. +# +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# +# texinfo_no_detailmenu = False + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'https://docs.python.org/': None, + 'http://pillow.readthedocs.io/en/3.2.x/': None, + 'http://docs.scipy.org/doc/numpy/': None} diff --git a/pydoc/index.rst b/pydoc/index.rst new file mode 100644 index 0000000..03ddda4 --- /dev/null +++ b/pydoc/index.rst @@ -0,0 +1,28 @@ +.. pyclstm documentation master file, created by + sphinx-quickstart on Thu Oct 6 12:57:42 2016. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to pyclstm's documentation! +=================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + + +API Reference +============= + +.. automodule:: pyclstm + :members: + :undoc-members: + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/pydoc/make.bat b/pydoc/make.bat new file mode 100644 index 0000000..62fe0f0 --- /dev/null +++ b/pydoc/make.bat @@ -0,0 +1,281 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +set I18NSPHINXOPTS=%SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. epub3 to make an epub3 + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. xml to make Docutils-native XML files + echo. pseudoxml to make pseudoxml-XML files for display purposes + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + echo. coverage to run coverage check of the documentation if enabled + echo. dummy to check syntax errors of document sources + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + + +REM Check if sphinx-build is available and fallback to Python version if any +%SPHINXBUILD% 1>NUL 2>NUL +if errorlevel 9009 goto sphinx_python +goto sphinx_ok + +:sphinx_python + +set SPHINXBUILD=python -m sphinx.__init__ +%SPHINXBUILD% 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +:sphinx_ok + + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\pyclstm.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\pyclstm.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "epub3" ( + %SPHINXBUILD% -b epub3 %ALLSPHINXOPTS% %BUILDDIR%/epub3 + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub3 file is in %BUILDDIR%/epub3. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdf" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdfja" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf-ja + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +if "%1" == "coverage" ( + %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage + if errorlevel 1 exit /b 1 + echo. + echo.Testing of coverage in the sources finished, look at the ^ +results in %BUILDDIR%/coverage/python.txt. + goto end +) + +if "%1" == "xml" ( + %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The XML files are in %BUILDDIR%/xml. + goto end +) + +if "%1" == "pseudoxml" ( + %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. + goto end +) + +if "%1" == "dummy" ( + %SPHINXBUILD% -b dummy %ALLSPHINXOPTS% %BUILDDIR%/dummy + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. Dummy builder generates no files. + goto end +) + +:end From 9aecfe0ffa2e8ac2faa56893258219ef264b9e43 Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Fri, 7 Oct 2016 22:51:02 +0200 Subject: [PATCH 18/34] Update README --- README.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 87c2f6d..ed4ab3f 100644 --- a/README.md +++ b/README.md @@ -68,10 +68,7 @@ After building the executables, you can run two simple test runs as follows: To build the Python extension, run - python setup.py build - sudo python setup.py install - -(this is currently broken) + pip install . # Documentation / Examples @@ -165,9 +162,10 @@ storage format. # Python API -The `clstm.i` file implements a simple Python interface to clstm, plus -a wrapper that makes an INetwork mostly a replacement for the lstm.py -implementation from ocropy. +The source code includes a Python interface to clstm (via Cython). Currently +it only exposes the `CLSTMOCR` class for OCR training and prediction. +To install it, just make sure you have the above dependencies and Cython +installed and run `pip install .`. # Comand Line Drivers From 3c9b5adbac92764ffc14b955292af2262afb6eee Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Fri, 14 Oct 2016 14:32:54 +0200 Subject: [PATCH 19/34] Rename to --- pyclstm.pyx | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyclstm.pyx b/pyclstm.pyx index 6589605..f4e45ca 100644 --- a/pyclstm.pyx +++ b/pyclstm.pyx @@ -119,13 +119,13 @@ cdef class ClstmOcr: if not rv: raise IOError("Could not save model to {}".format(fname)) - cpdef prepare_training(self, lexicon, int num_hidden=100, + cpdef prepare_training(self, graphemes, int num_hidden=100, float learning_rate=0.0001, float momentum=0.9): - """ Prepare training by setting the lexicon and hyperparameters. + """ Prepare training by setting the graphemes and hyperparameters. - :param lexicon: Iterable of characters that are to be recognized + :param graphemes: Iterable of graphemes that are to be recognized by the OCR model, must not have duplicates - :type lexicon: iterable of str/unicode + :type graphemes: iterable of str/unicode :param num_hidden: Number of hidden units in the LSTM layers, larger values require more storage/memory and take longer for training and recognition, so try to find @@ -136,12 +136,12 @@ cdef class ClstmOcr: :param momentum: Momentum for the model training :type momentum: float """ - lexicon_str = u"".join(sorted(lexicon)) + graphemes_str = u"".join(sorted(graphemes_str)) cdef vector[int] codec - cdef Py_ssize_t length = len(lexicon_str.encode("UTF-16")) // 2 + cdef Py_ssize_t length = len(graphemes_str.encode("UTF-16")) // 2 cdef wchar_t *wchars = malloc(length * sizeof(wchar_t)) cdef Py_ssize_t number_written = PyUnicode_AsWideChar( - lexicon_str, wchars, length) + graphemes_str, wchars, length) codec.push_back(0) for i in range(length-1): codec.push_back((wchars[i])) From 02f14b26ec4e115910956e8d27102149d9690a6e Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Sat, 3 Sep 2016 11:00:34 +0200 Subject: [PATCH 20/34] Don't track generated protobuf code --- clstm.pb.cc | 1407 --------------------------------------------------- clstm.pb.h | 1102 ---------------------------------------- 2 files changed, 2509 deletions(-) delete mode 100644 clstm.pb.cc delete mode 100644 clstm.pb.h diff --git a/clstm.pb.cc b/clstm.pb.cc deleted file mode 100644 index e6c1ce7..0000000 --- a/clstm.pb.cc +++ /dev/null @@ -1,1407 +0,0 @@ -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: clstm.proto - -#define INTERNAL_SUPPRESS_PROTOBUF_FIELD_DEPRECATION -#include "clstm.pb.h" - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -// @@protoc_insertion_point(includes) - -namespace clstm { - -namespace { - -const ::google::protobuf::Descriptor* KeyValue_descriptor_ = NULL; -const ::google::protobuf::internal::GeneratedMessageReflection* - KeyValue_reflection_ = NULL; -const ::google::protobuf::Descriptor* Array_descriptor_ = NULL; -const ::google::protobuf::internal::GeneratedMessageReflection* - Array_reflection_ = NULL; -const ::google::protobuf::Descriptor* NetworkProto_descriptor_ = NULL; -const ::google::protobuf::internal::GeneratedMessageReflection* - NetworkProto_reflection_ = NULL; - -} // namespace - - -void protobuf_AssignDesc_clstm_2eproto() { - protobuf_AddDesc_clstm_2eproto(); - const ::google::protobuf::FileDescriptor* file = - ::google::protobuf::DescriptorPool::generated_pool()->FindFileByName( - "clstm.proto"); - GOOGLE_CHECK(file != NULL); - KeyValue_descriptor_ = file->message_type(0); - static const int KeyValue_offsets_[2] = { - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(KeyValue, key_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(KeyValue, value_), - }; - KeyValue_reflection_ = - new ::google::protobuf::internal::GeneratedMessageReflection( - KeyValue_descriptor_, - KeyValue::default_instance_, - KeyValue_offsets_, - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(KeyValue, _has_bits_[0]), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(KeyValue, _unknown_fields_), - -1, - ::google::protobuf::DescriptorPool::generated_pool(), - ::google::protobuf::MessageFactory::generated_factory(), - sizeof(KeyValue)); - Array_descriptor_ = file->message_type(1); - static const int Array_offsets_[3] = { - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Array, name_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Array, dim_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Array, value_), - }; - Array_reflection_ = - new ::google::protobuf::internal::GeneratedMessageReflection( - Array_descriptor_, - Array::default_instance_, - Array_offsets_, - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Array, _has_bits_[0]), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Array, _unknown_fields_), - -1, - ::google::protobuf::DescriptorPool::generated_pool(), - ::google::protobuf::MessageFactory::generated_factory(), - sizeof(Array)); - NetworkProto_descriptor_ = file->message_type(2); - static const int NetworkProto_offsets_[9] = { - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(NetworkProto, kind_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(NetworkProto, name_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(NetworkProto, ninput_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(NetworkProto, noutput_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(NetworkProto, icodec_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(NetworkProto, codec_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(NetworkProto, attribute_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(NetworkProto, weights_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(NetworkProto, sub_), - }; - NetworkProto_reflection_ = - new ::google::protobuf::internal::GeneratedMessageReflection( - NetworkProto_descriptor_, - NetworkProto::default_instance_, - NetworkProto_offsets_, - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(NetworkProto, _has_bits_[0]), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(NetworkProto, _unknown_fields_), - -1, - ::google::protobuf::DescriptorPool::generated_pool(), - ::google::protobuf::MessageFactory::generated_factory(), - sizeof(NetworkProto)); -} - -namespace { - -GOOGLE_PROTOBUF_DECLARE_ONCE(protobuf_AssignDescriptors_once_); -inline void protobuf_AssignDescriptorsOnce() { - ::google::protobuf::GoogleOnceInit(&protobuf_AssignDescriptors_once_, - &protobuf_AssignDesc_clstm_2eproto); -} - -void protobuf_RegisterTypes(const ::std::string&) { - protobuf_AssignDescriptorsOnce(); - ::google::protobuf::MessageFactory::InternalRegisterGeneratedMessage( - KeyValue_descriptor_, &KeyValue::default_instance()); - ::google::protobuf::MessageFactory::InternalRegisterGeneratedMessage( - Array_descriptor_, &Array::default_instance()); - ::google::protobuf::MessageFactory::InternalRegisterGeneratedMessage( - NetworkProto_descriptor_, &NetworkProto::default_instance()); -} - -} // namespace - -void protobuf_ShutdownFile_clstm_2eproto() { - delete KeyValue::default_instance_; - delete KeyValue_reflection_; - delete Array::default_instance_; - delete Array_reflection_; - delete NetworkProto::default_instance_; - delete NetworkProto_reflection_; -} - -void protobuf_AddDesc_clstm_2eproto() { - static bool already_here = false; - if (already_here) return; - already_here = true; - GOOGLE_PROTOBUF_VERIFY_VERSION; - - ::google::protobuf::DescriptorPool::InternalAddGeneratedFile( - "\n\013clstm.proto\022\005clstm\"&\n\010KeyValue\022\013\n\003key\030" - "\001 \002(\t\022\r\n\005value\030\002 \002(\t\"1\n\005Array\022\014\n\004name\030\001 " - "\001(\t\022\013\n\003dim\030\002 \003(\005\022\r\n\005value\030\003 \003(\002\"\317\001\n\014Netw" - "orkProto\022\014\n\004kind\030\001 \002(\t\022\014\n\004name\030\002 \001(\t\022\016\n\006" - "ninput\030\n \002(\005\022\017\n\007noutput\030\013 \002(\005\022\016\n\006icodec\030" - "\014 \003(\005\022\r\n\005codec\030\r \003(\005\022\"\n\tattribute\030\024 \003(\0132" - "\017.clstm.KeyValue\022\035\n\007weights\030\036 \003(\0132\014.clst" - "m.Array\022 \n\003sub\030( \003(\0132\023.clstm.NetworkProt" - "o", 321); - ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile( - "clstm.proto", &protobuf_RegisterTypes); - KeyValue::default_instance_ = new KeyValue(); - Array::default_instance_ = new Array(); - NetworkProto::default_instance_ = new NetworkProto(); - KeyValue::default_instance_->InitAsDefaultInstance(); - Array::default_instance_->InitAsDefaultInstance(); - NetworkProto::default_instance_->InitAsDefaultInstance(); - ::google::protobuf::internal::OnShutdown(&protobuf_ShutdownFile_clstm_2eproto); -} - -// Force AddDescriptors() to be called at static initialization time. -struct StaticDescriptorInitializer_clstm_2eproto { - StaticDescriptorInitializer_clstm_2eproto() { - protobuf_AddDesc_clstm_2eproto(); - } -} static_descriptor_initializer_clstm_2eproto_; - -// =================================================================== - -#ifndef _MSC_VER -const int KeyValue::kKeyFieldNumber; -const int KeyValue::kValueFieldNumber; -#endif // !_MSC_VER - -KeyValue::KeyValue() - : ::google::protobuf::Message() { - SharedCtor(); - // @@protoc_insertion_point(constructor:clstm.KeyValue) -} - -void KeyValue::InitAsDefaultInstance() { -} - -KeyValue::KeyValue(const KeyValue& from) - : ::google::protobuf::Message() { - SharedCtor(); - MergeFrom(from); - // @@protoc_insertion_point(copy_constructor:clstm.KeyValue) -} - -void KeyValue::SharedCtor() { - ::google::protobuf::internal::GetEmptyString(); - _cached_size_ = 0; - key_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - value_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - ::memset(_has_bits_, 0, sizeof(_has_bits_)); -} - -KeyValue::~KeyValue() { - // @@protoc_insertion_point(destructor:clstm.KeyValue) - SharedDtor(); -} - -void KeyValue::SharedDtor() { - if (key_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - delete key_; - } - if (value_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - delete value_; - } - if (this != default_instance_) { - } -} - -void KeyValue::SetCachedSize(int size) const { - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); -} -const ::google::protobuf::Descriptor* KeyValue::descriptor() { - protobuf_AssignDescriptorsOnce(); - return KeyValue_descriptor_; -} - -const KeyValue& KeyValue::default_instance() { - if (default_instance_ == NULL) protobuf_AddDesc_clstm_2eproto(); - return *default_instance_; -} - -KeyValue* KeyValue::default_instance_ = NULL; - -KeyValue* KeyValue::New() const { - return new KeyValue; -} - -void KeyValue::Clear() { - if (_has_bits_[0 / 32] & 3) { - if (has_key()) { - if (key_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - key_->clear(); - } - } - if (has_value()) { - if (value_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - value_->clear(); - } - } - } - ::memset(_has_bits_, 0, sizeof(_has_bits_)); - mutable_unknown_fields()->Clear(); -} - -bool KeyValue::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure - ::google::protobuf::uint32 tag; - // @@protoc_insertion_point(parse_start:clstm.KeyValue) - for (;;) { - ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { - // required string key = 1; - case 1: { - if (tag == 10) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_key())); - ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( - this->key().data(), this->key().length(), - ::google::protobuf::internal::WireFormat::PARSE, - "key"); - } else { - goto handle_unusual; - } - if (input->ExpectTag(18)) goto parse_value; - break; - } - - // required string value = 2; - case 2: { - if (tag == 18) { - parse_value: - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_value())); - ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( - this->value().data(), this->value().length(), - ::google::protobuf::internal::WireFormat::PARSE, - "value"); - } else { - goto handle_unusual; - } - if (input->ExpectAtEnd()) goto success; - break; - } - - default: { - handle_unusual: - if (tag == 0 || - ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == - ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { - goto success; - } - DO_(::google::protobuf::internal::WireFormat::SkipField( - input, tag, mutable_unknown_fields())); - break; - } - } - } -success: - // @@protoc_insertion_point(parse_success:clstm.KeyValue) - return true; -failure: - // @@protoc_insertion_point(parse_failure:clstm.KeyValue) - return false; -#undef DO_ -} - -void KeyValue::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:clstm.KeyValue) - // required string key = 1; - if (has_key()) { - ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( - this->key().data(), this->key().length(), - ::google::protobuf::internal::WireFormat::SERIALIZE, - "key"); - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 1, this->key(), output); - } - - // required string value = 2; - if (has_value()) { - ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( - this->value().data(), this->value().length(), - ::google::protobuf::internal::WireFormat::SERIALIZE, - "value"); - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 2, this->value(), output); - } - - if (!unknown_fields().empty()) { - ::google::protobuf::internal::WireFormat::SerializeUnknownFields( - unknown_fields(), output); - } - // @@protoc_insertion_point(serialize_end:clstm.KeyValue) -} - -::google::protobuf::uint8* KeyValue::SerializeWithCachedSizesToArray( - ::google::protobuf::uint8* target) const { - // @@protoc_insertion_point(serialize_to_array_start:clstm.KeyValue) - // required string key = 1; - if (has_key()) { - ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( - this->key().data(), this->key().length(), - ::google::protobuf::internal::WireFormat::SERIALIZE, - "key"); - target = - ::google::protobuf::internal::WireFormatLite::WriteStringToArray( - 1, this->key(), target); - } - - // required string value = 2; - if (has_value()) { - ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( - this->value().data(), this->value().length(), - ::google::protobuf::internal::WireFormat::SERIALIZE, - "value"); - target = - ::google::protobuf::internal::WireFormatLite::WriteStringToArray( - 2, this->value(), target); - } - - if (!unknown_fields().empty()) { - target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray( - unknown_fields(), target); - } - // @@protoc_insertion_point(serialize_to_array_end:clstm.KeyValue) - return target; -} - -int KeyValue::ByteSize() const { - int total_size = 0; - - if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { - // required string key = 1; - if (has_key()) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->key()); - } - - // required string value = 2; - if (has_value()) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->value()); - } - - } - if (!unknown_fields().empty()) { - total_size += - ::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize( - unknown_fields()); - } - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = total_size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); - return total_size; -} - -void KeyValue::MergeFrom(const ::google::protobuf::Message& from) { - GOOGLE_CHECK_NE(&from, this); - const KeyValue* source = - ::google::protobuf::internal::dynamic_cast_if_available( - &from); - if (source == NULL) { - ::google::protobuf::internal::ReflectionOps::Merge(from, this); - } else { - MergeFrom(*source); - } -} - -void KeyValue::MergeFrom(const KeyValue& from) { - GOOGLE_CHECK_NE(&from, this); - if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { - if (from.has_key()) { - set_key(from.key()); - } - if (from.has_value()) { - set_value(from.value()); - } - } - mutable_unknown_fields()->MergeFrom(from.unknown_fields()); -} - -void KeyValue::CopyFrom(const ::google::protobuf::Message& from) { - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -void KeyValue::CopyFrom(const KeyValue& from) { - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -bool KeyValue::IsInitialized() const { - if ((_has_bits_[0] & 0x00000003) != 0x00000003) return false; - - return true; -} - -void KeyValue::Swap(KeyValue* other) { - if (other != this) { - std::swap(key_, other->key_); - std::swap(value_, other->value_); - std::swap(_has_bits_[0], other->_has_bits_[0]); - _unknown_fields_.Swap(&other->_unknown_fields_); - std::swap(_cached_size_, other->_cached_size_); - } -} - -::google::protobuf::Metadata KeyValue::GetMetadata() const { - protobuf_AssignDescriptorsOnce(); - ::google::protobuf::Metadata metadata; - metadata.descriptor = KeyValue_descriptor_; - metadata.reflection = KeyValue_reflection_; - return metadata; -} - - -// =================================================================== - -#ifndef _MSC_VER -const int Array::kNameFieldNumber; -const int Array::kDimFieldNumber; -const int Array::kValueFieldNumber; -#endif // !_MSC_VER - -Array::Array() - : ::google::protobuf::Message() { - SharedCtor(); - // @@protoc_insertion_point(constructor:clstm.Array) -} - -void Array::InitAsDefaultInstance() { -} - -Array::Array(const Array& from) - : ::google::protobuf::Message() { - SharedCtor(); - MergeFrom(from); - // @@protoc_insertion_point(copy_constructor:clstm.Array) -} - -void Array::SharedCtor() { - ::google::protobuf::internal::GetEmptyString(); - _cached_size_ = 0; - name_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - ::memset(_has_bits_, 0, sizeof(_has_bits_)); -} - -Array::~Array() { - // @@protoc_insertion_point(destructor:clstm.Array) - SharedDtor(); -} - -void Array::SharedDtor() { - if (name_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - delete name_; - } - if (this != default_instance_) { - } -} - -void Array::SetCachedSize(int size) const { - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); -} -const ::google::protobuf::Descriptor* Array::descriptor() { - protobuf_AssignDescriptorsOnce(); - return Array_descriptor_; -} - -const Array& Array::default_instance() { - if (default_instance_ == NULL) protobuf_AddDesc_clstm_2eproto(); - return *default_instance_; -} - -Array* Array::default_instance_ = NULL; - -Array* Array::New() const { - return new Array; -} - -void Array::Clear() { - if (has_name()) { - if (name_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - name_->clear(); - } - } - dim_.Clear(); - value_.Clear(); - ::memset(_has_bits_, 0, sizeof(_has_bits_)); - mutable_unknown_fields()->Clear(); -} - -bool Array::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure - ::google::protobuf::uint32 tag; - // @@protoc_insertion_point(parse_start:clstm.Array) - for (;;) { - ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { - // optional string name = 1; - case 1: { - if (tag == 10) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_name())); - ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( - this->name().data(), this->name().length(), - ::google::protobuf::internal::WireFormat::PARSE, - "name"); - } else { - goto handle_unusual; - } - if (input->ExpectTag(16)) goto parse_dim; - break; - } - - // repeated int32 dim = 2; - case 2: { - if (tag == 16) { - parse_dim: - DO_((::google::protobuf::internal::WireFormatLite::ReadRepeatedPrimitive< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - 1, 16, input, this->mutable_dim()))); - } else if (tag == 18) { - DO_((::google::protobuf::internal::WireFormatLite::ReadPackedPrimitiveNoInline< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - input, this->mutable_dim()))); - } else { - goto handle_unusual; - } - if (input->ExpectTag(16)) goto parse_dim; - if (input->ExpectTag(29)) goto parse_value; - break; - } - - // repeated float value = 3; - case 3: { - if (tag == 29) { - parse_value: - DO_((::google::protobuf::internal::WireFormatLite::ReadRepeatedPrimitive< - float, ::google::protobuf::internal::WireFormatLite::TYPE_FLOAT>( - 1, 29, input, this->mutable_value()))); - } else if (tag == 26) { - DO_((::google::protobuf::internal::WireFormatLite::ReadPackedPrimitiveNoInline< - float, ::google::protobuf::internal::WireFormatLite::TYPE_FLOAT>( - input, this->mutable_value()))); - } else { - goto handle_unusual; - } - if (input->ExpectTag(29)) goto parse_value; - if (input->ExpectAtEnd()) goto success; - break; - } - - default: { - handle_unusual: - if (tag == 0 || - ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == - ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { - goto success; - } - DO_(::google::protobuf::internal::WireFormat::SkipField( - input, tag, mutable_unknown_fields())); - break; - } - } - } -success: - // @@protoc_insertion_point(parse_success:clstm.Array) - return true; -failure: - // @@protoc_insertion_point(parse_failure:clstm.Array) - return false; -#undef DO_ -} - -void Array::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:clstm.Array) - // optional string name = 1; - if (has_name()) { - ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( - this->name().data(), this->name().length(), - ::google::protobuf::internal::WireFormat::SERIALIZE, - "name"); - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 1, this->name(), output); - } - - // repeated int32 dim = 2; - for (int i = 0; i < this->dim_size(); i++) { - ::google::protobuf::internal::WireFormatLite::WriteInt32( - 2, this->dim(i), output); - } - - // repeated float value = 3; - for (int i = 0; i < this->value_size(); i++) { - ::google::protobuf::internal::WireFormatLite::WriteFloat( - 3, this->value(i), output); - } - - if (!unknown_fields().empty()) { - ::google::protobuf::internal::WireFormat::SerializeUnknownFields( - unknown_fields(), output); - } - // @@protoc_insertion_point(serialize_end:clstm.Array) -} - -::google::protobuf::uint8* Array::SerializeWithCachedSizesToArray( - ::google::protobuf::uint8* target) const { - // @@protoc_insertion_point(serialize_to_array_start:clstm.Array) - // optional string name = 1; - if (has_name()) { - ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( - this->name().data(), this->name().length(), - ::google::protobuf::internal::WireFormat::SERIALIZE, - "name"); - target = - ::google::protobuf::internal::WireFormatLite::WriteStringToArray( - 1, this->name(), target); - } - - // repeated int32 dim = 2; - for (int i = 0; i < this->dim_size(); i++) { - target = ::google::protobuf::internal::WireFormatLite:: - WriteInt32ToArray(2, this->dim(i), target); - } - - // repeated float value = 3; - for (int i = 0; i < this->value_size(); i++) { - target = ::google::protobuf::internal::WireFormatLite:: - WriteFloatToArray(3, this->value(i), target); - } - - if (!unknown_fields().empty()) { - target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray( - unknown_fields(), target); - } - // @@protoc_insertion_point(serialize_to_array_end:clstm.Array) - return target; -} - -int Array::ByteSize() const { - int total_size = 0; - - if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { - // optional string name = 1; - if (has_name()) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->name()); - } - - } - // repeated int32 dim = 2; - { - int data_size = 0; - for (int i = 0; i < this->dim_size(); i++) { - data_size += ::google::protobuf::internal::WireFormatLite:: - Int32Size(this->dim(i)); - } - total_size += 1 * this->dim_size() + data_size; - } - - // repeated float value = 3; - { - int data_size = 0; - data_size = 4 * this->value_size(); - total_size += 1 * this->value_size() + data_size; - } - - if (!unknown_fields().empty()) { - total_size += - ::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize( - unknown_fields()); - } - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = total_size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); - return total_size; -} - -void Array::MergeFrom(const ::google::protobuf::Message& from) { - GOOGLE_CHECK_NE(&from, this); - const Array* source = - ::google::protobuf::internal::dynamic_cast_if_available( - &from); - if (source == NULL) { - ::google::protobuf::internal::ReflectionOps::Merge(from, this); - } else { - MergeFrom(*source); - } -} - -void Array::MergeFrom(const Array& from) { - GOOGLE_CHECK_NE(&from, this); - dim_.MergeFrom(from.dim_); - value_.MergeFrom(from.value_); - if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { - if (from.has_name()) { - set_name(from.name()); - } - } - mutable_unknown_fields()->MergeFrom(from.unknown_fields()); -} - -void Array::CopyFrom(const ::google::protobuf::Message& from) { - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -void Array::CopyFrom(const Array& from) { - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -bool Array::IsInitialized() const { - - return true; -} - -void Array::Swap(Array* other) { - if (other != this) { - std::swap(name_, other->name_); - dim_.Swap(&other->dim_); - value_.Swap(&other->value_); - std::swap(_has_bits_[0], other->_has_bits_[0]); - _unknown_fields_.Swap(&other->_unknown_fields_); - std::swap(_cached_size_, other->_cached_size_); - } -} - -::google::protobuf::Metadata Array::GetMetadata() const { - protobuf_AssignDescriptorsOnce(); - ::google::protobuf::Metadata metadata; - metadata.descriptor = Array_descriptor_; - metadata.reflection = Array_reflection_; - return metadata; -} - - -// =================================================================== - -#ifndef _MSC_VER -const int NetworkProto::kKindFieldNumber; -const int NetworkProto::kNameFieldNumber; -const int NetworkProto::kNinputFieldNumber; -const int NetworkProto::kNoutputFieldNumber; -const int NetworkProto::kIcodecFieldNumber; -const int NetworkProto::kCodecFieldNumber; -const int NetworkProto::kAttributeFieldNumber; -const int NetworkProto::kWeightsFieldNumber; -const int NetworkProto::kSubFieldNumber; -#endif // !_MSC_VER - -NetworkProto::NetworkProto() - : ::google::protobuf::Message() { - SharedCtor(); - // @@protoc_insertion_point(constructor:clstm.NetworkProto) -} - -void NetworkProto::InitAsDefaultInstance() { -} - -NetworkProto::NetworkProto(const NetworkProto& from) - : ::google::protobuf::Message() { - SharedCtor(); - MergeFrom(from); - // @@protoc_insertion_point(copy_constructor:clstm.NetworkProto) -} - -void NetworkProto::SharedCtor() { - ::google::protobuf::internal::GetEmptyString(); - _cached_size_ = 0; - kind_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - name_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - ninput_ = 0; - noutput_ = 0; - ::memset(_has_bits_, 0, sizeof(_has_bits_)); -} - -NetworkProto::~NetworkProto() { - // @@protoc_insertion_point(destructor:clstm.NetworkProto) - SharedDtor(); -} - -void NetworkProto::SharedDtor() { - if (kind_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - delete kind_; - } - if (name_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - delete name_; - } - if (this != default_instance_) { - } -} - -void NetworkProto::SetCachedSize(int size) const { - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); -} -const ::google::protobuf::Descriptor* NetworkProto::descriptor() { - protobuf_AssignDescriptorsOnce(); - return NetworkProto_descriptor_; -} - -const NetworkProto& NetworkProto::default_instance() { - if (default_instance_ == NULL) protobuf_AddDesc_clstm_2eproto(); - return *default_instance_; -} - -NetworkProto* NetworkProto::default_instance_ = NULL; - -NetworkProto* NetworkProto::New() const { - return new NetworkProto; -} - -void NetworkProto::Clear() { -#define OFFSET_OF_FIELD_(f) (reinterpret_cast( \ - &reinterpret_cast(16)->f) - \ - reinterpret_cast(16)) - -#define ZR_(first, last) do { \ - size_t f = OFFSET_OF_FIELD_(first); \ - size_t n = OFFSET_OF_FIELD_(last) - f + sizeof(last); \ - ::memset(&first, 0, n); \ - } while (0) - - if (_has_bits_[0 / 32] & 15) { - ZR_(ninput_, noutput_); - if (has_kind()) { - if (kind_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - kind_->clear(); - } - } - if (has_name()) { - if (name_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - name_->clear(); - } - } - } - -#undef OFFSET_OF_FIELD_ -#undef ZR_ - - icodec_.Clear(); - codec_.Clear(); - attribute_.Clear(); - weights_.Clear(); - sub_.Clear(); - ::memset(_has_bits_, 0, sizeof(_has_bits_)); - mutable_unknown_fields()->Clear(); -} - -bool NetworkProto::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure - ::google::protobuf::uint32 tag; - // @@protoc_insertion_point(parse_start:clstm.NetworkProto) - for (;;) { - ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(16383); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { - // required string kind = 1; - case 1: { - if (tag == 10) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_kind())); - ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( - this->kind().data(), this->kind().length(), - ::google::protobuf::internal::WireFormat::PARSE, - "kind"); - } else { - goto handle_unusual; - } - if (input->ExpectTag(18)) goto parse_name; - break; - } - - // optional string name = 2; - case 2: { - if (tag == 18) { - parse_name: - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_name())); - ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( - this->name().data(), this->name().length(), - ::google::protobuf::internal::WireFormat::PARSE, - "name"); - } else { - goto handle_unusual; - } - if (input->ExpectTag(80)) goto parse_ninput; - break; - } - - // required int32 ninput = 10; - case 10: { - if (tag == 80) { - parse_ninput: - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - input, &ninput_))); - set_has_ninput(); - } else { - goto handle_unusual; - } - if (input->ExpectTag(88)) goto parse_noutput; - break; - } - - // required int32 noutput = 11; - case 11: { - if (tag == 88) { - parse_noutput: - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - input, &noutput_))); - set_has_noutput(); - } else { - goto handle_unusual; - } - if (input->ExpectTag(96)) goto parse_icodec; - break; - } - - // repeated int32 icodec = 12; - case 12: { - if (tag == 96) { - parse_icodec: - DO_((::google::protobuf::internal::WireFormatLite::ReadRepeatedPrimitive< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - 1, 96, input, this->mutable_icodec()))); - } else if (tag == 98) { - DO_((::google::protobuf::internal::WireFormatLite::ReadPackedPrimitiveNoInline< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - input, this->mutable_icodec()))); - } else { - goto handle_unusual; - } - if (input->ExpectTag(96)) goto parse_icodec; - if (input->ExpectTag(104)) goto parse_codec; - break; - } - - // repeated int32 codec = 13; - case 13: { - if (tag == 104) { - parse_codec: - DO_((::google::protobuf::internal::WireFormatLite::ReadRepeatedPrimitive< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - 1, 104, input, this->mutable_codec()))); - } else if (tag == 106) { - DO_((::google::protobuf::internal::WireFormatLite::ReadPackedPrimitiveNoInline< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - input, this->mutable_codec()))); - } else { - goto handle_unusual; - } - if (input->ExpectTag(104)) goto parse_codec; - if (input->ExpectTag(162)) goto parse_attribute; - break; - } - - // repeated .clstm.KeyValue attribute = 20; - case 20: { - if (tag == 162) { - parse_attribute: - DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( - input, add_attribute())); - } else { - goto handle_unusual; - } - if (input->ExpectTag(162)) goto parse_attribute; - if (input->ExpectTag(242)) goto parse_weights; - break; - } - - // repeated .clstm.Array weights = 30; - case 30: { - if (tag == 242) { - parse_weights: - DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( - input, add_weights())); - } else { - goto handle_unusual; - } - if (input->ExpectTag(242)) goto parse_weights; - if (input->ExpectTag(322)) goto parse_sub; - break; - } - - // repeated .clstm.NetworkProto sub = 40; - case 40: { - if (tag == 322) { - parse_sub: - DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( - input, add_sub())); - } else { - goto handle_unusual; - } - if (input->ExpectTag(322)) goto parse_sub; - if (input->ExpectAtEnd()) goto success; - break; - } - - default: { - handle_unusual: - if (tag == 0 || - ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == - ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { - goto success; - } - DO_(::google::protobuf::internal::WireFormat::SkipField( - input, tag, mutable_unknown_fields())); - break; - } - } - } -success: - // @@protoc_insertion_point(parse_success:clstm.NetworkProto) - return true; -failure: - // @@protoc_insertion_point(parse_failure:clstm.NetworkProto) - return false; -#undef DO_ -} - -void NetworkProto::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:clstm.NetworkProto) - // required string kind = 1; - if (has_kind()) { - ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( - this->kind().data(), this->kind().length(), - ::google::protobuf::internal::WireFormat::SERIALIZE, - "kind"); - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 1, this->kind(), output); - } - - // optional string name = 2; - if (has_name()) { - ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( - this->name().data(), this->name().length(), - ::google::protobuf::internal::WireFormat::SERIALIZE, - "name"); - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 2, this->name(), output); - } - - // required int32 ninput = 10; - if (has_ninput()) { - ::google::protobuf::internal::WireFormatLite::WriteInt32(10, this->ninput(), output); - } - - // required int32 noutput = 11; - if (has_noutput()) { - ::google::protobuf::internal::WireFormatLite::WriteInt32(11, this->noutput(), output); - } - - // repeated int32 icodec = 12; - for (int i = 0; i < this->icodec_size(); i++) { - ::google::protobuf::internal::WireFormatLite::WriteInt32( - 12, this->icodec(i), output); - } - - // repeated int32 codec = 13; - for (int i = 0; i < this->codec_size(); i++) { - ::google::protobuf::internal::WireFormatLite::WriteInt32( - 13, this->codec(i), output); - } - - // repeated .clstm.KeyValue attribute = 20; - for (int i = 0; i < this->attribute_size(); i++) { - ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray( - 20, this->attribute(i), output); - } - - // repeated .clstm.Array weights = 30; - for (int i = 0; i < this->weights_size(); i++) { - ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray( - 30, this->weights(i), output); - } - - // repeated .clstm.NetworkProto sub = 40; - for (int i = 0; i < this->sub_size(); i++) { - ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray( - 40, this->sub(i), output); - } - - if (!unknown_fields().empty()) { - ::google::protobuf::internal::WireFormat::SerializeUnknownFields( - unknown_fields(), output); - } - // @@protoc_insertion_point(serialize_end:clstm.NetworkProto) -} - -::google::protobuf::uint8* NetworkProto::SerializeWithCachedSizesToArray( - ::google::protobuf::uint8* target) const { - // @@protoc_insertion_point(serialize_to_array_start:clstm.NetworkProto) - // required string kind = 1; - if (has_kind()) { - ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( - this->kind().data(), this->kind().length(), - ::google::protobuf::internal::WireFormat::SERIALIZE, - "kind"); - target = - ::google::protobuf::internal::WireFormatLite::WriteStringToArray( - 1, this->kind(), target); - } - - // optional string name = 2; - if (has_name()) { - ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( - this->name().data(), this->name().length(), - ::google::protobuf::internal::WireFormat::SERIALIZE, - "name"); - target = - ::google::protobuf::internal::WireFormatLite::WriteStringToArray( - 2, this->name(), target); - } - - // required int32 ninput = 10; - if (has_ninput()) { - target = ::google::protobuf::internal::WireFormatLite::WriteInt32ToArray(10, this->ninput(), target); - } - - // required int32 noutput = 11; - if (has_noutput()) { - target = ::google::protobuf::internal::WireFormatLite::WriteInt32ToArray(11, this->noutput(), target); - } - - // repeated int32 icodec = 12; - for (int i = 0; i < this->icodec_size(); i++) { - target = ::google::protobuf::internal::WireFormatLite:: - WriteInt32ToArray(12, this->icodec(i), target); - } - - // repeated int32 codec = 13; - for (int i = 0; i < this->codec_size(); i++) { - target = ::google::protobuf::internal::WireFormatLite:: - WriteInt32ToArray(13, this->codec(i), target); - } - - // repeated .clstm.KeyValue attribute = 20; - for (int i = 0; i < this->attribute_size(); i++) { - target = ::google::protobuf::internal::WireFormatLite:: - WriteMessageNoVirtualToArray( - 20, this->attribute(i), target); - } - - // repeated .clstm.Array weights = 30; - for (int i = 0; i < this->weights_size(); i++) { - target = ::google::protobuf::internal::WireFormatLite:: - WriteMessageNoVirtualToArray( - 30, this->weights(i), target); - } - - // repeated .clstm.NetworkProto sub = 40; - for (int i = 0; i < this->sub_size(); i++) { - target = ::google::protobuf::internal::WireFormatLite:: - WriteMessageNoVirtualToArray( - 40, this->sub(i), target); - } - - if (!unknown_fields().empty()) { - target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray( - unknown_fields(), target); - } - // @@protoc_insertion_point(serialize_to_array_end:clstm.NetworkProto) - return target; -} - -int NetworkProto::ByteSize() const { - int total_size = 0; - - if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { - // required string kind = 1; - if (has_kind()) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->kind()); - } - - // optional string name = 2; - if (has_name()) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->name()); - } - - // required int32 ninput = 10; - if (has_ninput()) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::Int32Size( - this->ninput()); - } - - // required int32 noutput = 11; - if (has_noutput()) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::Int32Size( - this->noutput()); - } - - } - // repeated int32 icodec = 12; - { - int data_size = 0; - for (int i = 0; i < this->icodec_size(); i++) { - data_size += ::google::protobuf::internal::WireFormatLite:: - Int32Size(this->icodec(i)); - } - total_size += 1 * this->icodec_size() + data_size; - } - - // repeated int32 codec = 13; - { - int data_size = 0; - for (int i = 0; i < this->codec_size(); i++) { - data_size += ::google::protobuf::internal::WireFormatLite:: - Int32Size(this->codec(i)); - } - total_size += 1 * this->codec_size() + data_size; - } - - // repeated .clstm.KeyValue attribute = 20; - total_size += 2 * this->attribute_size(); - for (int i = 0; i < this->attribute_size(); i++) { - total_size += - ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( - this->attribute(i)); - } - - // repeated .clstm.Array weights = 30; - total_size += 2 * this->weights_size(); - for (int i = 0; i < this->weights_size(); i++) { - total_size += - ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( - this->weights(i)); - } - - // repeated .clstm.NetworkProto sub = 40; - total_size += 2 * this->sub_size(); - for (int i = 0; i < this->sub_size(); i++) { - total_size += - ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( - this->sub(i)); - } - - if (!unknown_fields().empty()) { - total_size += - ::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize( - unknown_fields()); - } - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = total_size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); - return total_size; -} - -void NetworkProto::MergeFrom(const ::google::protobuf::Message& from) { - GOOGLE_CHECK_NE(&from, this); - const NetworkProto* source = - ::google::protobuf::internal::dynamic_cast_if_available( - &from); - if (source == NULL) { - ::google::protobuf::internal::ReflectionOps::Merge(from, this); - } else { - MergeFrom(*source); - } -} - -void NetworkProto::MergeFrom(const NetworkProto& from) { - GOOGLE_CHECK_NE(&from, this); - icodec_.MergeFrom(from.icodec_); - codec_.MergeFrom(from.codec_); - attribute_.MergeFrom(from.attribute_); - weights_.MergeFrom(from.weights_); - sub_.MergeFrom(from.sub_); - if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { - if (from.has_kind()) { - set_kind(from.kind()); - } - if (from.has_name()) { - set_name(from.name()); - } - if (from.has_ninput()) { - set_ninput(from.ninput()); - } - if (from.has_noutput()) { - set_noutput(from.noutput()); - } - } - mutable_unknown_fields()->MergeFrom(from.unknown_fields()); -} - -void NetworkProto::CopyFrom(const ::google::protobuf::Message& from) { - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -void NetworkProto::CopyFrom(const NetworkProto& from) { - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -bool NetworkProto::IsInitialized() const { - if ((_has_bits_[0] & 0x0000000d) != 0x0000000d) return false; - - if (!::google::protobuf::internal::AllAreInitialized(this->attribute())) return false; - if (!::google::protobuf::internal::AllAreInitialized(this->sub())) return false; - return true; -} - -void NetworkProto::Swap(NetworkProto* other) { - if (other != this) { - std::swap(kind_, other->kind_); - std::swap(name_, other->name_); - std::swap(ninput_, other->ninput_); - std::swap(noutput_, other->noutput_); - icodec_.Swap(&other->icodec_); - codec_.Swap(&other->codec_); - attribute_.Swap(&other->attribute_); - weights_.Swap(&other->weights_); - sub_.Swap(&other->sub_); - std::swap(_has_bits_[0], other->_has_bits_[0]); - _unknown_fields_.Swap(&other->_unknown_fields_); - std::swap(_cached_size_, other->_cached_size_); - } -} - -::google::protobuf::Metadata NetworkProto::GetMetadata() const { - protobuf_AssignDescriptorsOnce(); - ::google::protobuf::Metadata metadata; - metadata.descriptor = NetworkProto_descriptor_; - metadata.reflection = NetworkProto_reflection_; - return metadata; -} - - -// @@protoc_insertion_point(namespace_scope) - -} // namespace clstm - -// @@protoc_insertion_point(global_scope) diff --git a/clstm.pb.h b/clstm.pb.h deleted file mode 100644 index 9d7f627..0000000 --- a/clstm.pb.h +++ /dev/null @@ -1,1102 +0,0 @@ -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: clstm.proto - -#ifndef PROTOBUF_clstm_2eproto__INCLUDED -#define PROTOBUF_clstm_2eproto__INCLUDED - -#include - -#include - -#if GOOGLE_PROTOBUF_VERSION < 2006000 -#error This file was generated by a newer version of protoc which is -#error incompatible with your Protocol Buffer headers. Please update -#error your headers. -#endif -#if 2006001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION -#error This file was generated by an older version of protoc which is -#error incompatible with your Protocol Buffer headers. Please -#error regenerate this file with a newer version of protoc. -#endif - -#include -#include -#include -#include -#include -// @@protoc_insertion_point(includes) - -namespace clstm { - -// Internal implementation detail -- do not call these. -void protobuf_AddDesc_clstm_2eproto(); -void protobuf_AssignDesc_clstm_2eproto(); -void protobuf_ShutdownFile_clstm_2eproto(); - -class KeyValue; -class Array; -class NetworkProto; - -// =================================================================== - -class KeyValue : public ::google::protobuf::Message { - public: - KeyValue(); - virtual ~KeyValue(); - - KeyValue(const KeyValue& from); - - inline KeyValue& operator=(const KeyValue& from) { - CopyFrom(from); - return *this; - } - - inline const ::google::protobuf::UnknownFieldSet& unknown_fields() const { - return _unknown_fields_; - } - - inline ::google::protobuf::UnknownFieldSet* mutable_unknown_fields() { - return &_unknown_fields_; - } - - static const ::google::protobuf::Descriptor* descriptor(); - static const KeyValue& default_instance(); - - void Swap(KeyValue* other); - - // implements Message ---------------------------------------------- - - KeyValue* New() const; - void CopyFrom(const ::google::protobuf::Message& from); - void MergeFrom(const ::google::protobuf::Message& from); - void CopyFrom(const KeyValue& from); - void MergeFrom(const KeyValue& from); - void Clear(); - bool IsInitialized() const; - - int ByteSize() const; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input); - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const; - ::google::protobuf::uint8* SerializeWithCachedSizesToArray(::google::protobuf::uint8* output) const; - int GetCachedSize() const { return _cached_size_; } - private: - void SharedCtor(); - void SharedDtor(); - void SetCachedSize(int size) const; - public: - ::google::protobuf::Metadata GetMetadata() const; - - // nested types ---------------------------------------------------- - - // accessors ------------------------------------------------------- - - // required string key = 1; - inline bool has_key() const; - inline void clear_key(); - static const int kKeyFieldNumber = 1; - inline const ::std::string& key() const; - inline void set_key(const ::std::string& value); - inline void set_key(const char* value); - inline void set_key(const char* value, size_t size); - inline ::std::string* mutable_key(); - inline ::std::string* release_key(); - inline void set_allocated_key(::std::string* key); - - // required string value = 2; - inline bool has_value() const; - inline void clear_value(); - static const int kValueFieldNumber = 2; - inline const ::std::string& value() const; - inline void set_value(const ::std::string& value); - inline void set_value(const char* value); - inline void set_value(const char* value, size_t size); - inline ::std::string* mutable_value(); - inline ::std::string* release_value(); - inline void set_allocated_value(::std::string* value); - - // @@protoc_insertion_point(class_scope:clstm.KeyValue) - private: - inline void set_has_key(); - inline void clear_has_key(); - inline void set_has_value(); - inline void clear_has_value(); - - ::google::protobuf::UnknownFieldSet _unknown_fields_; - - ::google::protobuf::uint32 _has_bits_[1]; - mutable int _cached_size_; - ::std::string* key_; - ::std::string* value_; - friend void protobuf_AddDesc_clstm_2eproto(); - friend void protobuf_AssignDesc_clstm_2eproto(); - friend void protobuf_ShutdownFile_clstm_2eproto(); - - void InitAsDefaultInstance(); - static KeyValue* default_instance_; -}; -// ------------------------------------------------------------------- - -class Array : public ::google::protobuf::Message { - public: - Array(); - virtual ~Array(); - - Array(const Array& from); - - inline Array& operator=(const Array& from) { - CopyFrom(from); - return *this; - } - - inline const ::google::protobuf::UnknownFieldSet& unknown_fields() const { - return _unknown_fields_; - } - - inline ::google::protobuf::UnknownFieldSet* mutable_unknown_fields() { - return &_unknown_fields_; - } - - static const ::google::protobuf::Descriptor* descriptor(); - static const Array& default_instance(); - - void Swap(Array* other); - - // implements Message ---------------------------------------------- - - Array* New() const; - void CopyFrom(const ::google::protobuf::Message& from); - void MergeFrom(const ::google::protobuf::Message& from); - void CopyFrom(const Array& from); - void MergeFrom(const Array& from); - void Clear(); - bool IsInitialized() const; - - int ByteSize() const; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input); - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const; - ::google::protobuf::uint8* SerializeWithCachedSizesToArray(::google::protobuf::uint8* output) const; - int GetCachedSize() const { return _cached_size_; } - private: - void SharedCtor(); - void SharedDtor(); - void SetCachedSize(int size) const; - public: - ::google::protobuf::Metadata GetMetadata() const; - - // nested types ---------------------------------------------------- - - // accessors ------------------------------------------------------- - - // optional string name = 1; - inline bool has_name() const; - inline void clear_name(); - static const int kNameFieldNumber = 1; - inline const ::std::string& name() const; - inline void set_name(const ::std::string& value); - inline void set_name(const char* value); - inline void set_name(const char* value, size_t size); - inline ::std::string* mutable_name(); - inline ::std::string* release_name(); - inline void set_allocated_name(::std::string* name); - - // repeated int32 dim = 2; - inline int dim_size() const; - inline void clear_dim(); - static const int kDimFieldNumber = 2; - inline ::google::protobuf::int32 dim(int index) const; - inline void set_dim(int index, ::google::protobuf::int32 value); - inline void add_dim(::google::protobuf::int32 value); - inline const ::google::protobuf::RepeatedField< ::google::protobuf::int32 >& - dim() const; - inline ::google::protobuf::RepeatedField< ::google::protobuf::int32 >* - mutable_dim(); - - // repeated float value = 3; - inline int value_size() const; - inline void clear_value(); - static const int kValueFieldNumber = 3; - inline float value(int index) const; - inline void set_value(int index, float value); - inline void add_value(float value); - inline const ::google::protobuf::RepeatedField< float >& - value() const; - inline ::google::protobuf::RepeatedField< float >* - mutable_value(); - - // @@protoc_insertion_point(class_scope:clstm.Array) - private: - inline void set_has_name(); - inline void clear_has_name(); - - ::google::protobuf::UnknownFieldSet _unknown_fields_; - - ::google::protobuf::uint32 _has_bits_[1]; - mutable int _cached_size_; - ::std::string* name_; - ::google::protobuf::RepeatedField< ::google::protobuf::int32 > dim_; - ::google::protobuf::RepeatedField< float > value_; - friend void protobuf_AddDesc_clstm_2eproto(); - friend void protobuf_AssignDesc_clstm_2eproto(); - friend void protobuf_ShutdownFile_clstm_2eproto(); - - void InitAsDefaultInstance(); - static Array* default_instance_; -}; -// ------------------------------------------------------------------- - -class NetworkProto : public ::google::protobuf::Message { - public: - NetworkProto(); - virtual ~NetworkProto(); - - NetworkProto(const NetworkProto& from); - - inline NetworkProto& operator=(const NetworkProto& from) { - CopyFrom(from); - return *this; - } - - inline const ::google::protobuf::UnknownFieldSet& unknown_fields() const { - return _unknown_fields_; - } - - inline ::google::protobuf::UnknownFieldSet* mutable_unknown_fields() { - return &_unknown_fields_; - } - - static const ::google::protobuf::Descriptor* descriptor(); - static const NetworkProto& default_instance(); - - void Swap(NetworkProto* other); - - // implements Message ---------------------------------------------- - - NetworkProto* New() const; - void CopyFrom(const ::google::protobuf::Message& from); - void MergeFrom(const ::google::protobuf::Message& from); - void CopyFrom(const NetworkProto& from); - void MergeFrom(const NetworkProto& from); - void Clear(); - bool IsInitialized() const; - - int ByteSize() const; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input); - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const; - ::google::protobuf::uint8* SerializeWithCachedSizesToArray(::google::protobuf::uint8* output) const; - int GetCachedSize() const { return _cached_size_; } - private: - void SharedCtor(); - void SharedDtor(); - void SetCachedSize(int size) const; - public: - ::google::protobuf::Metadata GetMetadata() const; - - // nested types ---------------------------------------------------- - - // accessors ------------------------------------------------------- - - // required string kind = 1; - inline bool has_kind() const; - inline void clear_kind(); - static const int kKindFieldNumber = 1; - inline const ::std::string& kind() const; - inline void set_kind(const ::std::string& value); - inline void set_kind(const char* value); - inline void set_kind(const char* value, size_t size); - inline ::std::string* mutable_kind(); - inline ::std::string* release_kind(); - inline void set_allocated_kind(::std::string* kind); - - // optional string name = 2; - inline bool has_name() const; - inline void clear_name(); - static const int kNameFieldNumber = 2; - inline const ::std::string& name() const; - inline void set_name(const ::std::string& value); - inline void set_name(const char* value); - inline void set_name(const char* value, size_t size); - inline ::std::string* mutable_name(); - inline ::std::string* release_name(); - inline void set_allocated_name(::std::string* name); - - // required int32 ninput = 10; - inline bool has_ninput() const; - inline void clear_ninput(); - static const int kNinputFieldNumber = 10; - inline ::google::protobuf::int32 ninput() const; - inline void set_ninput(::google::protobuf::int32 value); - - // required int32 noutput = 11; - inline bool has_noutput() const; - inline void clear_noutput(); - static const int kNoutputFieldNumber = 11; - inline ::google::protobuf::int32 noutput() const; - inline void set_noutput(::google::protobuf::int32 value); - - // repeated int32 icodec = 12; - inline int icodec_size() const; - inline void clear_icodec(); - static const int kIcodecFieldNumber = 12; - inline ::google::protobuf::int32 icodec(int index) const; - inline void set_icodec(int index, ::google::protobuf::int32 value); - inline void add_icodec(::google::protobuf::int32 value); - inline const ::google::protobuf::RepeatedField< ::google::protobuf::int32 >& - icodec() const; - inline ::google::protobuf::RepeatedField< ::google::protobuf::int32 >* - mutable_icodec(); - - // repeated int32 codec = 13; - inline int codec_size() const; - inline void clear_codec(); - static const int kCodecFieldNumber = 13; - inline ::google::protobuf::int32 codec(int index) const; - inline void set_codec(int index, ::google::protobuf::int32 value); - inline void add_codec(::google::protobuf::int32 value); - inline const ::google::protobuf::RepeatedField< ::google::protobuf::int32 >& - codec() const; - inline ::google::protobuf::RepeatedField< ::google::protobuf::int32 >* - mutable_codec(); - - // repeated .clstm.KeyValue attribute = 20; - inline int attribute_size() const; - inline void clear_attribute(); - static const int kAttributeFieldNumber = 20; - inline const ::clstm::KeyValue& attribute(int index) const; - inline ::clstm::KeyValue* mutable_attribute(int index); - inline ::clstm::KeyValue* add_attribute(); - inline const ::google::protobuf::RepeatedPtrField< ::clstm::KeyValue >& - attribute() const; - inline ::google::protobuf::RepeatedPtrField< ::clstm::KeyValue >* - mutable_attribute(); - - // repeated .clstm.Array weights = 30; - inline int weights_size() const; - inline void clear_weights(); - static const int kWeightsFieldNumber = 30; - inline const ::clstm::Array& weights(int index) const; - inline ::clstm::Array* mutable_weights(int index); - inline ::clstm::Array* add_weights(); - inline const ::google::protobuf::RepeatedPtrField< ::clstm::Array >& - weights() const; - inline ::google::protobuf::RepeatedPtrField< ::clstm::Array >* - mutable_weights(); - - // repeated .clstm.NetworkProto sub = 40; - inline int sub_size() const; - inline void clear_sub(); - static const int kSubFieldNumber = 40; - inline const ::clstm::NetworkProto& sub(int index) const; - inline ::clstm::NetworkProto* mutable_sub(int index); - inline ::clstm::NetworkProto* add_sub(); - inline const ::google::protobuf::RepeatedPtrField< ::clstm::NetworkProto >& - sub() const; - inline ::google::protobuf::RepeatedPtrField< ::clstm::NetworkProto >* - mutable_sub(); - - // @@protoc_insertion_point(class_scope:clstm.NetworkProto) - private: - inline void set_has_kind(); - inline void clear_has_kind(); - inline void set_has_name(); - inline void clear_has_name(); - inline void set_has_ninput(); - inline void clear_has_ninput(); - inline void set_has_noutput(); - inline void clear_has_noutput(); - - ::google::protobuf::UnknownFieldSet _unknown_fields_; - - ::google::protobuf::uint32 _has_bits_[1]; - mutable int _cached_size_; - ::std::string* kind_; - ::std::string* name_; - ::google::protobuf::int32 ninput_; - ::google::protobuf::int32 noutput_; - ::google::protobuf::RepeatedField< ::google::protobuf::int32 > icodec_; - ::google::protobuf::RepeatedField< ::google::protobuf::int32 > codec_; - ::google::protobuf::RepeatedPtrField< ::clstm::KeyValue > attribute_; - ::google::protobuf::RepeatedPtrField< ::clstm::Array > weights_; - ::google::protobuf::RepeatedPtrField< ::clstm::NetworkProto > sub_; - friend void protobuf_AddDesc_clstm_2eproto(); - friend void protobuf_AssignDesc_clstm_2eproto(); - friend void protobuf_ShutdownFile_clstm_2eproto(); - - void InitAsDefaultInstance(); - static NetworkProto* default_instance_; -}; -// =================================================================== - - -// =================================================================== - -// KeyValue - -// required string key = 1; -inline bool KeyValue::has_key() const { - return (_has_bits_[0] & 0x00000001u) != 0; -} -inline void KeyValue::set_has_key() { - _has_bits_[0] |= 0x00000001u; -} -inline void KeyValue::clear_has_key() { - _has_bits_[0] &= ~0x00000001u; -} -inline void KeyValue::clear_key() { - if (key_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - key_->clear(); - } - clear_has_key(); -} -inline const ::std::string& KeyValue::key() const { - // @@protoc_insertion_point(field_get:clstm.KeyValue.key) - return *key_; -} -inline void KeyValue::set_key(const ::std::string& value) { - set_has_key(); - if (key_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - key_ = new ::std::string; - } - key_->assign(value); - // @@protoc_insertion_point(field_set:clstm.KeyValue.key) -} -inline void KeyValue::set_key(const char* value) { - set_has_key(); - if (key_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - key_ = new ::std::string; - } - key_->assign(value); - // @@protoc_insertion_point(field_set_char:clstm.KeyValue.key) -} -inline void KeyValue::set_key(const char* value, size_t size) { - set_has_key(); - if (key_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - key_ = new ::std::string; - } - key_->assign(reinterpret_cast(value), size); - // @@protoc_insertion_point(field_set_pointer:clstm.KeyValue.key) -} -inline ::std::string* KeyValue::mutable_key() { - set_has_key(); - if (key_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - key_ = new ::std::string; - } - // @@protoc_insertion_point(field_mutable:clstm.KeyValue.key) - return key_; -} -inline ::std::string* KeyValue::release_key() { - clear_has_key(); - if (key_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - return NULL; - } else { - ::std::string* temp = key_; - key_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - return temp; - } -} -inline void KeyValue::set_allocated_key(::std::string* key) { - if (key_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - delete key_; - } - if (key) { - set_has_key(); - key_ = key; - } else { - clear_has_key(); - key_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - } - // @@protoc_insertion_point(field_set_allocated:clstm.KeyValue.key) -} - -// required string value = 2; -inline bool KeyValue::has_value() const { - return (_has_bits_[0] & 0x00000002u) != 0; -} -inline void KeyValue::set_has_value() { - _has_bits_[0] |= 0x00000002u; -} -inline void KeyValue::clear_has_value() { - _has_bits_[0] &= ~0x00000002u; -} -inline void KeyValue::clear_value() { - if (value_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - value_->clear(); - } - clear_has_value(); -} -inline const ::std::string& KeyValue::value() const { - // @@protoc_insertion_point(field_get:clstm.KeyValue.value) - return *value_; -} -inline void KeyValue::set_value(const ::std::string& value) { - set_has_value(); - if (value_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - value_ = new ::std::string; - } - value_->assign(value); - // @@protoc_insertion_point(field_set:clstm.KeyValue.value) -} -inline void KeyValue::set_value(const char* value) { - set_has_value(); - if (value_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - value_ = new ::std::string; - } - value_->assign(value); - // @@protoc_insertion_point(field_set_char:clstm.KeyValue.value) -} -inline void KeyValue::set_value(const char* value, size_t size) { - set_has_value(); - if (value_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - value_ = new ::std::string; - } - value_->assign(reinterpret_cast(value), size); - // @@protoc_insertion_point(field_set_pointer:clstm.KeyValue.value) -} -inline ::std::string* KeyValue::mutable_value() { - set_has_value(); - if (value_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - value_ = new ::std::string; - } - // @@protoc_insertion_point(field_mutable:clstm.KeyValue.value) - return value_; -} -inline ::std::string* KeyValue::release_value() { - clear_has_value(); - if (value_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - return NULL; - } else { - ::std::string* temp = value_; - value_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - return temp; - } -} -inline void KeyValue::set_allocated_value(::std::string* value) { - if (value_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - delete value_; - } - if (value) { - set_has_value(); - value_ = value; - } else { - clear_has_value(); - value_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - } - // @@protoc_insertion_point(field_set_allocated:clstm.KeyValue.value) -} - -// ------------------------------------------------------------------- - -// Array - -// optional string name = 1; -inline bool Array::has_name() const { - return (_has_bits_[0] & 0x00000001u) != 0; -} -inline void Array::set_has_name() { - _has_bits_[0] |= 0x00000001u; -} -inline void Array::clear_has_name() { - _has_bits_[0] &= ~0x00000001u; -} -inline void Array::clear_name() { - if (name_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - name_->clear(); - } - clear_has_name(); -} -inline const ::std::string& Array::name() const { - // @@protoc_insertion_point(field_get:clstm.Array.name) - return *name_; -} -inline void Array::set_name(const ::std::string& value) { - set_has_name(); - if (name_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - name_ = new ::std::string; - } - name_->assign(value); - // @@protoc_insertion_point(field_set:clstm.Array.name) -} -inline void Array::set_name(const char* value) { - set_has_name(); - if (name_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - name_ = new ::std::string; - } - name_->assign(value); - // @@protoc_insertion_point(field_set_char:clstm.Array.name) -} -inline void Array::set_name(const char* value, size_t size) { - set_has_name(); - if (name_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - name_ = new ::std::string; - } - name_->assign(reinterpret_cast(value), size); - // @@protoc_insertion_point(field_set_pointer:clstm.Array.name) -} -inline ::std::string* Array::mutable_name() { - set_has_name(); - if (name_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - name_ = new ::std::string; - } - // @@protoc_insertion_point(field_mutable:clstm.Array.name) - return name_; -} -inline ::std::string* Array::release_name() { - clear_has_name(); - if (name_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - return NULL; - } else { - ::std::string* temp = name_; - name_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - return temp; - } -} -inline void Array::set_allocated_name(::std::string* name) { - if (name_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - delete name_; - } - if (name) { - set_has_name(); - name_ = name; - } else { - clear_has_name(); - name_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - } - // @@protoc_insertion_point(field_set_allocated:clstm.Array.name) -} - -// repeated int32 dim = 2; -inline int Array::dim_size() const { - return dim_.size(); -} -inline void Array::clear_dim() { - dim_.Clear(); -} -inline ::google::protobuf::int32 Array::dim(int index) const { - // @@protoc_insertion_point(field_get:clstm.Array.dim) - return dim_.Get(index); -} -inline void Array::set_dim(int index, ::google::protobuf::int32 value) { - dim_.Set(index, value); - // @@protoc_insertion_point(field_set:clstm.Array.dim) -} -inline void Array::add_dim(::google::protobuf::int32 value) { - dim_.Add(value); - // @@protoc_insertion_point(field_add:clstm.Array.dim) -} -inline const ::google::protobuf::RepeatedField< ::google::protobuf::int32 >& -Array::dim() const { - // @@protoc_insertion_point(field_list:clstm.Array.dim) - return dim_; -} -inline ::google::protobuf::RepeatedField< ::google::protobuf::int32 >* -Array::mutable_dim() { - // @@protoc_insertion_point(field_mutable_list:clstm.Array.dim) - return &dim_; -} - -// repeated float value = 3; -inline int Array::value_size() const { - return value_.size(); -} -inline void Array::clear_value() { - value_.Clear(); -} -inline float Array::value(int index) const { - // @@protoc_insertion_point(field_get:clstm.Array.value) - return value_.Get(index); -} -inline void Array::set_value(int index, float value) { - value_.Set(index, value); - // @@protoc_insertion_point(field_set:clstm.Array.value) -} -inline void Array::add_value(float value) { - value_.Add(value); - // @@protoc_insertion_point(field_add:clstm.Array.value) -} -inline const ::google::protobuf::RepeatedField< float >& -Array::value() const { - // @@protoc_insertion_point(field_list:clstm.Array.value) - return value_; -} -inline ::google::protobuf::RepeatedField< float >* -Array::mutable_value() { - // @@protoc_insertion_point(field_mutable_list:clstm.Array.value) - return &value_; -} - -// ------------------------------------------------------------------- - -// NetworkProto - -// required string kind = 1; -inline bool NetworkProto::has_kind() const { - return (_has_bits_[0] & 0x00000001u) != 0; -} -inline void NetworkProto::set_has_kind() { - _has_bits_[0] |= 0x00000001u; -} -inline void NetworkProto::clear_has_kind() { - _has_bits_[0] &= ~0x00000001u; -} -inline void NetworkProto::clear_kind() { - if (kind_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - kind_->clear(); - } - clear_has_kind(); -} -inline const ::std::string& NetworkProto::kind() const { - // @@protoc_insertion_point(field_get:clstm.NetworkProto.kind) - return *kind_; -} -inline void NetworkProto::set_kind(const ::std::string& value) { - set_has_kind(); - if (kind_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - kind_ = new ::std::string; - } - kind_->assign(value); - // @@protoc_insertion_point(field_set:clstm.NetworkProto.kind) -} -inline void NetworkProto::set_kind(const char* value) { - set_has_kind(); - if (kind_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - kind_ = new ::std::string; - } - kind_->assign(value); - // @@protoc_insertion_point(field_set_char:clstm.NetworkProto.kind) -} -inline void NetworkProto::set_kind(const char* value, size_t size) { - set_has_kind(); - if (kind_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - kind_ = new ::std::string; - } - kind_->assign(reinterpret_cast(value), size); - // @@protoc_insertion_point(field_set_pointer:clstm.NetworkProto.kind) -} -inline ::std::string* NetworkProto::mutable_kind() { - set_has_kind(); - if (kind_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - kind_ = new ::std::string; - } - // @@protoc_insertion_point(field_mutable:clstm.NetworkProto.kind) - return kind_; -} -inline ::std::string* NetworkProto::release_kind() { - clear_has_kind(); - if (kind_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - return NULL; - } else { - ::std::string* temp = kind_; - kind_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - return temp; - } -} -inline void NetworkProto::set_allocated_kind(::std::string* kind) { - if (kind_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - delete kind_; - } - if (kind) { - set_has_kind(); - kind_ = kind; - } else { - clear_has_kind(); - kind_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - } - // @@protoc_insertion_point(field_set_allocated:clstm.NetworkProto.kind) -} - -// optional string name = 2; -inline bool NetworkProto::has_name() const { - return (_has_bits_[0] & 0x00000002u) != 0; -} -inline void NetworkProto::set_has_name() { - _has_bits_[0] |= 0x00000002u; -} -inline void NetworkProto::clear_has_name() { - _has_bits_[0] &= ~0x00000002u; -} -inline void NetworkProto::clear_name() { - if (name_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - name_->clear(); - } - clear_has_name(); -} -inline const ::std::string& NetworkProto::name() const { - // @@protoc_insertion_point(field_get:clstm.NetworkProto.name) - return *name_; -} -inline void NetworkProto::set_name(const ::std::string& value) { - set_has_name(); - if (name_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - name_ = new ::std::string; - } - name_->assign(value); - // @@protoc_insertion_point(field_set:clstm.NetworkProto.name) -} -inline void NetworkProto::set_name(const char* value) { - set_has_name(); - if (name_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - name_ = new ::std::string; - } - name_->assign(value); - // @@protoc_insertion_point(field_set_char:clstm.NetworkProto.name) -} -inline void NetworkProto::set_name(const char* value, size_t size) { - set_has_name(); - if (name_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - name_ = new ::std::string; - } - name_->assign(reinterpret_cast(value), size); - // @@protoc_insertion_point(field_set_pointer:clstm.NetworkProto.name) -} -inline ::std::string* NetworkProto::mutable_name() { - set_has_name(); - if (name_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - name_ = new ::std::string; - } - // @@protoc_insertion_point(field_mutable:clstm.NetworkProto.name) - return name_; -} -inline ::std::string* NetworkProto::release_name() { - clear_has_name(); - if (name_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - return NULL; - } else { - ::std::string* temp = name_; - name_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - return temp; - } -} -inline void NetworkProto::set_allocated_name(::std::string* name) { - if (name_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { - delete name_; - } - if (name) { - set_has_name(); - name_ = name; - } else { - clear_has_name(); - name_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - } - // @@protoc_insertion_point(field_set_allocated:clstm.NetworkProto.name) -} - -// required int32 ninput = 10; -inline bool NetworkProto::has_ninput() const { - return (_has_bits_[0] & 0x00000004u) != 0; -} -inline void NetworkProto::set_has_ninput() { - _has_bits_[0] |= 0x00000004u; -} -inline void NetworkProto::clear_has_ninput() { - _has_bits_[0] &= ~0x00000004u; -} -inline void NetworkProto::clear_ninput() { - ninput_ = 0; - clear_has_ninput(); -} -inline ::google::protobuf::int32 NetworkProto::ninput() const { - // @@protoc_insertion_point(field_get:clstm.NetworkProto.ninput) - return ninput_; -} -inline void NetworkProto::set_ninput(::google::protobuf::int32 value) { - set_has_ninput(); - ninput_ = value; - // @@protoc_insertion_point(field_set:clstm.NetworkProto.ninput) -} - -// required int32 noutput = 11; -inline bool NetworkProto::has_noutput() const { - return (_has_bits_[0] & 0x00000008u) != 0; -} -inline void NetworkProto::set_has_noutput() { - _has_bits_[0] |= 0x00000008u; -} -inline void NetworkProto::clear_has_noutput() { - _has_bits_[0] &= ~0x00000008u; -} -inline void NetworkProto::clear_noutput() { - noutput_ = 0; - clear_has_noutput(); -} -inline ::google::protobuf::int32 NetworkProto::noutput() const { - // @@protoc_insertion_point(field_get:clstm.NetworkProto.noutput) - return noutput_; -} -inline void NetworkProto::set_noutput(::google::protobuf::int32 value) { - set_has_noutput(); - noutput_ = value; - // @@protoc_insertion_point(field_set:clstm.NetworkProto.noutput) -} - -// repeated int32 icodec = 12; -inline int NetworkProto::icodec_size() const { - return icodec_.size(); -} -inline void NetworkProto::clear_icodec() { - icodec_.Clear(); -} -inline ::google::protobuf::int32 NetworkProto::icodec(int index) const { - // @@protoc_insertion_point(field_get:clstm.NetworkProto.icodec) - return icodec_.Get(index); -} -inline void NetworkProto::set_icodec(int index, ::google::protobuf::int32 value) { - icodec_.Set(index, value); - // @@protoc_insertion_point(field_set:clstm.NetworkProto.icodec) -} -inline void NetworkProto::add_icodec(::google::protobuf::int32 value) { - icodec_.Add(value); - // @@protoc_insertion_point(field_add:clstm.NetworkProto.icodec) -} -inline const ::google::protobuf::RepeatedField< ::google::protobuf::int32 >& -NetworkProto::icodec() const { - // @@protoc_insertion_point(field_list:clstm.NetworkProto.icodec) - return icodec_; -} -inline ::google::protobuf::RepeatedField< ::google::protobuf::int32 >* -NetworkProto::mutable_icodec() { - // @@protoc_insertion_point(field_mutable_list:clstm.NetworkProto.icodec) - return &icodec_; -} - -// repeated int32 codec = 13; -inline int NetworkProto::codec_size() const { - return codec_.size(); -} -inline void NetworkProto::clear_codec() { - codec_.Clear(); -} -inline ::google::protobuf::int32 NetworkProto::codec(int index) const { - // @@protoc_insertion_point(field_get:clstm.NetworkProto.codec) - return codec_.Get(index); -} -inline void NetworkProto::set_codec(int index, ::google::protobuf::int32 value) { - codec_.Set(index, value); - // @@protoc_insertion_point(field_set:clstm.NetworkProto.codec) -} -inline void NetworkProto::add_codec(::google::protobuf::int32 value) { - codec_.Add(value); - // @@protoc_insertion_point(field_add:clstm.NetworkProto.codec) -} -inline const ::google::protobuf::RepeatedField< ::google::protobuf::int32 >& -NetworkProto::codec() const { - // @@protoc_insertion_point(field_list:clstm.NetworkProto.codec) - return codec_; -} -inline ::google::protobuf::RepeatedField< ::google::protobuf::int32 >* -NetworkProto::mutable_codec() { - // @@protoc_insertion_point(field_mutable_list:clstm.NetworkProto.codec) - return &codec_; -} - -// repeated .clstm.KeyValue attribute = 20; -inline int NetworkProto::attribute_size() const { - return attribute_.size(); -} -inline void NetworkProto::clear_attribute() { - attribute_.Clear(); -} -inline const ::clstm::KeyValue& NetworkProto::attribute(int index) const { - // @@protoc_insertion_point(field_get:clstm.NetworkProto.attribute) - return attribute_.Get(index); -} -inline ::clstm::KeyValue* NetworkProto::mutable_attribute(int index) { - // @@protoc_insertion_point(field_mutable:clstm.NetworkProto.attribute) - return attribute_.Mutable(index); -} -inline ::clstm::KeyValue* NetworkProto::add_attribute() { - // @@protoc_insertion_point(field_add:clstm.NetworkProto.attribute) - return attribute_.Add(); -} -inline const ::google::protobuf::RepeatedPtrField< ::clstm::KeyValue >& -NetworkProto::attribute() const { - // @@protoc_insertion_point(field_list:clstm.NetworkProto.attribute) - return attribute_; -} -inline ::google::protobuf::RepeatedPtrField< ::clstm::KeyValue >* -NetworkProto::mutable_attribute() { - // @@protoc_insertion_point(field_mutable_list:clstm.NetworkProto.attribute) - return &attribute_; -} - -// repeated .clstm.Array weights = 30; -inline int NetworkProto::weights_size() const { - return weights_.size(); -} -inline void NetworkProto::clear_weights() { - weights_.Clear(); -} -inline const ::clstm::Array& NetworkProto::weights(int index) const { - // @@protoc_insertion_point(field_get:clstm.NetworkProto.weights) - return weights_.Get(index); -} -inline ::clstm::Array* NetworkProto::mutable_weights(int index) { - // @@protoc_insertion_point(field_mutable:clstm.NetworkProto.weights) - return weights_.Mutable(index); -} -inline ::clstm::Array* NetworkProto::add_weights() { - // @@protoc_insertion_point(field_add:clstm.NetworkProto.weights) - return weights_.Add(); -} -inline const ::google::protobuf::RepeatedPtrField< ::clstm::Array >& -NetworkProto::weights() const { - // @@protoc_insertion_point(field_list:clstm.NetworkProto.weights) - return weights_; -} -inline ::google::protobuf::RepeatedPtrField< ::clstm::Array >* -NetworkProto::mutable_weights() { - // @@protoc_insertion_point(field_mutable_list:clstm.NetworkProto.weights) - return &weights_; -} - -// repeated .clstm.NetworkProto sub = 40; -inline int NetworkProto::sub_size() const { - return sub_.size(); -} -inline void NetworkProto::clear_sub() { - sub_.Clear(); -} -inline const ::clstm::NetworkProto& NetworkProto::sub(int index) const { - // @@protoc_insertion_point(field_get:clstm.NetworkProto.sub) - return sub_.Get(index); -} -inline ::clstm::NetworkProto* NetworkProto::mutable_sub(int index) { - // @@protoc_insertion_point(field_mutable:clstm.NetworkProto.sub) - return sub_.Mutable(index); -} -inline ::clstm::NetworkProto* NetworkProto::add_sub() { - // @@protoc_insertion_point(field_add:clstm.NetworkProto.sub) - return sub_.Add(); -} -inline const ::google::protobuf::RepeatedPtrField< ::clstm::NetworkProto >& -NetworkProto::sub() const { - // @@protoc_insertion_point(field_list:clstm.NetworkProto.sub) - return sub_; -} -inline ::google::protobuf::RepeatedPtrField< ::clstm::NetworkProto >* -NetworkProto::mutable_sub() { - // @@protoc_insertion_point(field_mutable_list:clstm.NetworkProto.sub) - return &sub_; -} - - -// @@protoc_insertion_point(namespace_scope) - -} // namespace clstm - -#ifndef SWIG -namespace google { -namespace protobuf { - - -} // namespace google -} // namespace protobuf -#endif // SWIG - -// @@protoc_insertion_point(global_scope) - -#endif // PROTOBUF_clstm_2eproto__INCLUDED From 4701ccbffbebaa3ae71b981426bf68bbf36533a3 Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Fri, 14 Oct 2016 14:56:22 +0200 Subject: [PATCH 21/34] Fix typo in Cython code --- pyclstm.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyclstm.pyx b/pyclstm.pyx index f4e45ca..2ba2882 100644 --- a/pyclstm.pyx +++ b/pyclstm.pyx @@ -136,7 +136,7 @@ cdef class ClstmOcr: :param momentum: Momentum for the model training :type momentum: float """ - graphemes_str = u"".join(sorted(graphemes_str)) + graphemes_str = u"".join(sorted(graphemes)) cdef vector[int] codec cdef Py_ssize_t length = len(graphemes_str.encode("UTF-16")) // 2 cdef wchar_t *wchars = malloc(length * sizeof(wchar_t)) From fa6b00ebfedb8381a363d23916a0a72cf635317d Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Fri, 14 Oct 2016 19:02:43 +0200 Subject: [PATCH 22/34] Adapt run_uw3_500.py script --- run_uw3_500.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_uw3_500.py b/run_uw3_500.py index 43db75a..dc62e2f 100644 --- a/run_uw3_500.py +++ b/run_uw3_500.py @@ -16,8 +16,8 @@ test_data = all_data[400:] ocr = pyclstm.ClstmOcr() -ocr.create_bidi(set(chain.from_iterable(all_texts)), 100) -ocr.set_learning_rate(0.0001, 0.9) +graphemes = set(chain.from_iterable(all_texts)) +ocr.prepare_training(graphemes) for i in range(2000000): best_error = 1. From eed8b552808581a97428d85a91328b2c4eccee77 Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Fri, 14 Oct 2016 19:04:20 +0200 Subject: [PATCH 23/34] Update docs --- pydoc/conf.py | 5 +++-- pydoc/index.rst | 53 +++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/pydoc/conf.py b/pydoc/conf.py index f225876..3379c6a 100644 --- a/pydoc/conf.py +++ b/pydoc/conf.py @@ -19,6 +19,7 @@ # import os # import sys # sys.path.insert(0, os.path.abspath('.')) +import sphinx_rtd_theme # -- General configuration ------------------------------------------------ @@ -122,7 +123,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'default' +html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -131,7 +132,7 @@ # html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # The name for this set of Sphinx documents. # " v documentation" by default. diff --git a/pydoc/index.rst b/pydoc/index.rst index 03ddda4..429cd75 100644 --- a/pydoc/index.rst +++ b/pydoc/index.rst @@ -3,17 +3,58 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to pyclstm's documentation! -=================================== +pyclstm +======= -Contents: +Installation +------------ -.. toctree:: - :maxdepth: 2 +**Requirements:** + - A recent version of Eigen (>= 3.3) with development headers + - A C++ compiler (g++ is recommended) + - Cython + +**Installation:** + +.. code:: + + $ pip install git+https://github.com/jbaiter/clstm.git@cython + + +Example Usage +------------- + +**Training:** + +Refer to ``run_uw3_500.py`` in the root directory for a more comprehensive +example. + +.. code:: + + import pyclstm + ocr = pyclstm.ClstmOcr() + ocr.prepare_training( + graphemes=graphemes, # A list of characters the engine is supposed to recognize + ) + + # line_img can be an image loaded with PIL/Pillow or a numpy array + for line_img, ground_truth in training_data: + ocr.train(line_img, ground_truth) + ocr.save("my_model.clstm") + + +**Recognition:** + +.. code:: + + import pyclstm + ocr = pyclstm.ClstmOcr() + ocr.load("my_model.clstm") + text = ocr.recognize(line_img) API Reference -============= +------------- .. automodule:: pyclstm :members: From d96195a6a36676cd438f938b4667e2231fed2944 Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Mon, 10 Oct 2016 18:44:30 +0200 Subject: [PATCH 24/34] Fix typo in docstring --- pyclstm.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyclstm.pyx b/pyclstm.pyx index 2ba2882..13fb644 100644 --- a/pyclstm.pyx +++ b/pyclstm.pyx @@ -22,7 +22,7 @@ cdef extern from "Python.h": cpdef double levenshtein(unicode a, unicode b): - """ Determine the Levenshtein-distance between to unicode strings. + """ Determine the Levenshtein-distance between two unicode strings. :type a: unicode :type b: unicode From 002c9cea64540fca109964bee6ce2319566b3d09 Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Tue, 18 Oct 2016 21:32:33 +0200 Subject: [PATCH 25/34] Python 3 compatibility --- _clstm.pxd | 6 ++++++ pyclstm.pyx | 4 ++-- pyextra_defs.h | 18 ++++++++++++++++++ run_uw3_500.py | 8 ++++++-- setup.py | 4 +++- 5 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 pyextra_defs.h diff --git a/_clstm.pxd b/_clstm.pxd index b96acdc..9dae05c 100644 --- a/_clstm.pxd +++ b/_clstm.pxd @@ -1,3 +1,4 @@ +from cpython.ref cimport PyObject from libc.stddef cimport wchar_t from libcpp.vector cimport vector from libcpp.string cimport string @@ -18,6 +19,11 @@ cdef extern from "" namespace "std": iterator end() +cdef extern from "pyextra_defs.h": + cdef Py_ssize_t Unicode_AsWideChar(PyObject* ustr, Py_ssize_t length, + wchar_t* wchars) + + cdef extern from "pstring.h": wstring utf8_to_utf32(string s) diff --git a/pyclstm.pyx b/pyclstm.pyx index 13fb644..bcfe471 100644 --- a/pyclstm.pyx +++ b/pyclstm.pyx @@ -140,8 +140,8 @@ cdef class ClstmOcr: cdef vector[int] codec cdef Py_ssize_t length = len(graphemes_str.encode("UTF-16")) // 2 cdef wchar_t *wchars = malloc(length * sizeof(wchar_t)) - cdef Py_ssize_t number_written = PyUnicode_AsWideChar( - graphemes_str, wchars, length) + cdef Py_ssize_t number_written = _clstm.Unicode_AsWideChar( + graphemes_str, length, wchars) codec.push_back(0) for i in range(length-1): codec.push_back((wchars[i])) diff --git a/pyextra_defs.h b/pyextra_defs.h new file mode 100644 index 0000000..0b5ed86 --- /dev/null +++ b/pyextra_defs.h @@ -0,0 +1,18 @@ +#include + +/** This extra header is needed to make the code Python 3 compatible. + * The reason for this is that the signature of `PyUnicode_AsWideChar` changed + * due to the removal of the `PyUnicodeObject` (`unicode`) type in Python 3. + * + * Unfortunately Cython doesn't support conditional compilation out of the + * box, so we need to use this workaround. + **/ +#if PY_MAJOR_VERSION >= 3 +typedef PyObject UnicodeObject; +#else +typedef PyUnicodeObject UnicodeObject; +#endif + +Py_ssize_t Unicode_AsWideChar(PyObject* str, Py_ssize_t length, wchar_t *wchars) { + return PyUnicode_AsWideChar((UnicodeObject*) str, wchars, length); +} diff --git a/run_uw3_500.py b/run_uw3_500.py index dc62e2f..24276af 100644 --- a/run_uw3_500.py +++ b/run_uw3_500.py @@ -2,15 +2,19 @@ import glob import random +import sys from itertools import chain import pyclstm from PIL import Image all_imgs = [Image.open(p) for p in sorted(glob.glob("./book/*/*.png"))] -all_texts = [open(p).read().strip().decode('utf8') +all_texts = [open(p).read().strip() for p in sorted(glob.glob("./book/*/*.gt.txt"))] -all_data = zip(all_imgs, all_texts) +if sys.version <= (3,): + all_texts = [t.decode('utf8') for t in all_texts] + +all_data = list(zip(all_imgs, all_texts)) train_data = all_data[:400] test_data = all_data[400:] diff --git a/setup.py b/setup.py index 2b0bbcc..0c68759 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import os from distutils.core import setup, Extension @@ -9,7 +11,7 @@ def ensure_protobuf(): stale = (exists and os.path.getctime("clstm.pb.cc") < os.path.getctime("clstm.proto")) if not exists or stale: - print "Generating proto file" + print("Generating proto file") os.system("protoc clstm.proto --cpp_out=.") From 262b691dac545c38f9ad0028c0a916660bbe6ab2 Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Mon, 24 Oct 2016 16:51:33 +0200 Subject: [PATCH 26/34] Add requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ed5195b --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +Cython>=0.20 From 7050e5742b98fff555b50a3d5e49f11d09aa8c90 Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Mon, 24 Oct 2016 16:55:36 +0200 Subject: [PATCH 27/34] Fix std=c++11 flag in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0c68759..a6d2a4d 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ def ensure_protobuf(): '/usr/include/hdf5/serial', '/usr/include/hdf5'], libraries=['protobuf', 'png'], language='c++', - extra_compile_args=['-w', '-std=c++11', '-Wno-unused-result', '-g', + extra_compile_args=['-w', '--std=c++11', '-Wno-unused-result', '-g', '-DNODISPLAY=1', '-DTHROW=throw', '-DNDEBUG', '-Ofast', '-DEIGEN_NO_DEBUG', '-finline', '-ffast-math', '-fno-signaling-nans', '-funsafe-math-optimizations', From e67030886af382c7a7a8f32b28551117c7bc17fc Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Tue, 25 Oct 2016 14:19:44 +0200 Subject: [PATCH 28/34] run_uw3_500: sys.version{,_info}, die helpfully unless './book' --- run_uw3_500.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/run_uw3_500.py b/run_uw3_500.py index 24276af..ee126b2 100644 --- a/run_uw3_500.py +++ b/run_uw3_500.py @@ -1,6 +1,7 @@ from __future__ import division import glob +import os import random import sys from itertools import chain @@ -8,10 +9,15 @@ import pyclstm from PIL import Image +if not os.path.isdir('./book'): + print("Please download the sample data:\n\t" + "curl -L http://tmbdev.net/ocrdata/uw3-500.tgz|tar xf") + sys.exit(1) + all_imgs = [Image.open(p) for p in sorted(glob.glob("./book/*/*.png"))] all_texts = [open(p).read().strip() for p in sorted(glob.glob("./book/*/*.gt.txt"))] -if sys.version <= (3,): +if sys.version_info <= (3,): all_texts = [t.decode('utf8') for t in all_texts] all_data = list(zip(all_imgs, all_texts)) From 62dadea3b5618aa4c02e8e027ac7473471da8565 Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Tue, 25 Oct 2016 16:40:11 +0200 Subject: [PATCH 29/34] Allow all possible string types for fname in save/load --- pyclstm.pyx | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/pyclstm.pyx b/pyclstm.pyx index bcfe471..2fbd1eb 100644 --- a/pyclstm.pyx +++ b/pyclstm.pyx @@ -1,3 +1,6 @@ +import sys + +from cpython.version cimport PY_MAJOR_VERSION from cpython.ref cimport PyObject from libc.stddef cimport wchar_t from libc.stdlib cimport malloc, free @@ -32,6 +35,16 @@ cpdef double levenshtein(unicode a, unicode b): a.encode('utf8'), b.encode('utf8')) +cdef safe_str(in_str): + cdef bint needs_encode = ( + (PY_MAJOR_VERSION < 3 and not isinstance(in_str, str)) or + not isinstance(in_str, bytes)) + if needs_encode: + return in_str.encode('utf8') + else: + return in_str + + cdef load_img(img, _clstm.Tensor2 *data): """ Copy image data from a PIL image to an Eigen tensor. @@ -89,32 +102,35 @@ cdef class ClstmOcr: """ cdef _clstm.CLSTMOCR *_ocr - def __cinit__(self, str fname=None): + def __cinit__(self, fname=None): """ Initialize the OCR engine, optionally loading a model from disk. :param fname: Path to pre-trained model on disk - :type fname: str + :type fname: str/unicode/bytes """ self._ocr = new _clstm.CLSTMOCR() if fname: + fname = safe_str(fname) self.load(fname) - cpdef load(self, str fname): + cpdef load(self, fname): """ Load a pre-trained model from disk. :param fname: Path to pre-trained model on disk - :type fname: str + :type fname: str/unicode/bytes """ + fname = safe_str(fname) cdef bint rv = self._ocr.maybe_load(fname) if not rv: raise IOError("Could not load model from {}".format(fname)) - cpdef save(self, str fname): + cpdef save(self, fname): """ Save the model to disk. :param fname: Path to store model in - :type fname: str + :type fname: str/unicode/bytes """ + fname = safe_str(fname) cdef bint rv = self._ocr.maybe_save(fname) if not rv: raise IOError("Could not save model to {}".format(fname)) @@ -161,7 +177,7 @@ cdef class ClstmOcr: :param img: The line image for the ground truth :type img: :py:class:`PIL.Image.Image`/:py:class:`numpy.ndarray` :param text: The ground truth text for the line image - :type text: unicode + :type text: unicode/str :returns: The recognized text for the line image, can be used to estimate error against the ground truth (via :py:func:`levenshtein`) From d3ee308d792ec1c3428f1f563cefc30cbe16d30c Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Tue, 25 Oct 2016 16:47:48 +0200 Subject: [PATCH 30/34] Update required Cython version --- README.md | 4 ++-- pydoc/index.rst | 2 +- requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ed4ab3f..e8adff3 100644 --- a/README.md +++ b/README.md @@ -164,8 +164,8 @@ storage format. The source code includes a Python interface to clstm (via Cython). Currently it only exposes the `CLSTMOCR` class for OCR training and prediction. -To install it, just make sure you have the above dependencies and Cython -installed and run `pip install .`. +To install it, just make sure you have the above dependencies and +Cython (>=0.23) installed and run `pip install .`. # Comand Line Drivers diff --git a/pydoc/index.rst b/pydoc/index.rst index 429cd75..7096414 100644 --- a/pydoc/index.rst +++ b/pydoc/index.rst @@ -12,7 +12,7 @@ Installation **Requirements:** - A recent version of Eigen (>= 3.3) with development headers - A C++ compiler (g++ is recommended) - - Cython + - Cython (>= 0.23) **Installation:** diff --git a/requirements.txt b/requirements.txt index ed5195b..8ccc63c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -Cython>=0.20 +Cython>=0.23 From c690c997f07e8739e342568a8ea5a727f7d4a7a7 Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Tue, 25 Oct 2016 17:19:34 +0200 Subject: [PATCH 31/34] Remove unused import --- pyclstm.pyx | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyclstm.pyx b/pyclstm.pyx index 2fbd1eb..da77742 100644 --- a/pyclstm.pyx +++ b/pyclstm.pyx @@ -1,5 +1,3 @@ -import sys - from cpython.version cimport PY_MAJOR_VERSION from cpython.ref cimport PyObject from libc.stddef cimport wchar_t From 1c89c0af9308612d72b79939daecf189aa81990f Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Wed, 26 Oct 2016 11:04:20 +0200 Subject: [PATCH 32/34] Make image loading compatible with Pillow<2.9.0 --- pyclstm.pyx | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/pyclstm.pyx b/pyclstm.pyx index da77742..f89ad91 100644 --- a/pyclstm.pyx +++ b/pyclstm.pyx @@ -49,10 +49,14 @@ cdef load_img(img, _clstm.Tensor2 *data): :param img: Image :type img: :py:class:`PIL.Image.Image` """ - data.resize(img.width, img.height) + if hasattr(img, 'width'): + width, height = img.width, img.height + elif hasattr(img, 'size'): + width, height = img.size + data.resize(width, height) imgdata = img.load() - for i in range(img.width): - for j in range(img.height): + for i in range(width): + for j in range(height): px = imgdata[i, j] # Pillow returns pixels as [0, 255], but we need [0, 1] if isinstance(px, tuple): @@ -182,10 +186,10 @@ cdef class ClstmOcr: :rtype: unicode """ cdef _clstm.Tensor2 data - if hasattr(img, 'width'): - load_img(img, &data) - elif hasattr(img, 'shape'): + if hasattr(img, 'shape'): load_nparray(img, &data) + else: + load_img(img, &data) return self._ocr.train_utf8( data.map(), text.encode('utf8')).decode('utf8') @@ -198,10 +202,10 @@ cdef class ClstmOcr: :rtype: unicode """ cdef _clstm.Tensor2 data - if hasattr(img, 'width'): - load_img(img, &data) - elif hasattr(img, 'shape'): + if hasattr(img, 'shape'): load_nparray(img, &data) + else: + load_img(img, &data) return self._ocr.predict_utf8(data.map()).decode('utf8') def recognize_chars(self, img): @@ -218,10 +222,10 @@ cdef class ClstmOcr: cdef vector[_clstm.CharPrediction] preds cdef vector[_clstm.CharPrediction].iterator pred_it cdef wchar_t[2] cur_char - if hasattr(img, 'width'): - load_img(img, &data) - elif hasattr(img, 'shape'): + if hasattr(img, 'shape'): load_nparray(img, &data) + else: + load_img(img, &data) self._ocr.predict(preds, data.map()) for i in range(preds.size()): cur_char[0] = preds[i].c From 47653f30760f18b9a32563461d3858af69c98596 Mon Sep 17 00:00:00 2001 From: Johannes Baiter Date: Wed, 26 Oct 2016 17:11:35 +0200 Subject: [PATCH 33/34] Fix length calculation (thanks @mittagessen) --- pyclstm.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyclstm.pyx b/pyclstm.pyx index f89ad91..3a8633a 100644 --- a/pyclstm.pyx +++ b/pyclstm.pyx @@ -156,7 +156,7 @@ cdef class ClstmOcr: """ graphemes_str = u"".join(sorted(graphemes)) cdef vector[int] codec - cdef Py_ssize_t length = len(graphemes_str.encode("UTF-16")) // 2 + cdef Py_ssize_t length = len(graphemes_str) cdef wchar_t *wchars = malloc(length * sizeof(wchar_t)) cdef Py_ssize_t number_written = _clstm.Unicode_AsWideChar( graphemes_str, length, wchars) From 40cd27756511d3fe780be0e17aa1eb0d434c2715 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Thu, 27 Oct 2016 10:22:37 +0200 Subject: [PATCH 34/34] Basic training CLI 'pyclstm-train' --- pyclstm_cli/__init__.py | 1 + pyclstm_cli/train.py | 219 ++++++++++++++++++++++++++++++++++++++++ setup.py | 6 ++ 3 files changed, 226 insertions(+) create mode 100644 pyclstm_cli/__init__.py create mode 100644 pyclstm_cli/train.py diff --git a/pyclstm_cli/__init__.py b/pyclstm_cli/__init__.py new file mode 100644 index 0000000..fdff37f --- /dev/null +++ b/pyclstm_cli/__init__.py @@ -0,0 +1 @@ +from .train import OcrTrainer diff --git a/pyclstm_cli/train.py b/pyclstm_cli/train.py new file mode 100644 index 0000000..3a699dc --- /dev/null +++ b/pyclstm_cli/train.py @@ -0,0 +1,219 @@ +from __future__ import division + +from itertools import chain +import argparse +import glob +import logging +import os +import random +import re +import sys + +import pyclstm +from PIL import Image + +logging.basicConfig(level=logging.INFO, format="%(asctime)s\t%(message)s") + +class OcrTrainer: + + def __init__(self, + imgdir, + gtdir=None, + model_name=None, + train_ratio=.8, + max_iterations=2000000, + num_hidden=50, + learning_rate=0.0001, + momentum=0.9): + """Initialize a trainer. + + :param imgdir: Basedirectory containing '*/*.bin.png' images + :type imgdir: str/unicode + :param max_iterations: Maximum iterations + :type max_iterations: int + :param gtdir: Base directory containing '*/*.gt.txt' transcriptions + :type imgdir: str/unicode + :param train_ratio: Ratio of ground truth touse for training + :type train_ratio: float + :param model_name: Model basename. 'foo' => 'foo.clstm' or 'foo-1000.clstm' + :type model_name: str/unicode + :param num_hidden: See pyclstm.pyx + :param learning_rate: See pyclstm.pyx + :param momentum: See pyclstm.pyx + """ + if not gtdir: + gtdir = imgdir + self.model_name = model_name if model_name else re.sub(r"[^A-Za-z0-9_-]", "", imgdir) + self.imgdir = imgdir + all_imgs = [Image.open(p) + for p in sorted(glob.glob("{0}/*/*.bin.png".format(imgdir))) + ] + all_texts = [open(p).read().strip() + for p in sorted(glob.glob("{0}/*/*.gt.txt".format(gtdir)))] + if sys.version_info <= (3,): + all_texts = [t.decode('utf8') for t in all_texts] + all_data = list(zip(all_imgs, all_texts)) + nr_train = int(len(all_data) * train_ratio) + self.train_data = all_data[:nr_train] + self.test_data = all_data[nr_train:] + self.graphemes = set(chain.from_iterable(all_texts)) + self.ocr = pyclstm.ClstmOcr() + self.max_iterations = max_iterations + self.num_hidden = num_hidden + self.learning_rate = learning_rate + self.momentum = momentum + + def save_model(self, suffix=''): + """ + Save the model + + :param suffix: Optional suffix to save intermediary results. + :param suffix: str/unicode + """ + fname = self.model_name + if suffix != '': fname += '-' + suffix + fname += '.clstm' + logging.info("Saving model to {}".format(fname)) + self.ocr.save(fname) + + def calculate_error(self): + """ + Calculate error rate by recognizing test_data and comparing by + levenshtein distance + """ + logging.info("Calculating error") + errors = 0 + chars = 0 + for img, txt in self.test_data: + out = self.ocr.recognize(img) + errors += pyclstm.levenshtein(txt, out) + chars += len(txt) + return errors / chars + + def train(self, + log_out=True, + log_every=10, + save_every=-1, + calculate_every=1000): + """Train from OCR ground truth data. + + :param log_out: Whether to log 'OUT' + :param log_every: Log 'TRN'/'ALN' every N iterations + :param save_every: Save every N iterations with suffix -N.clst + :param calculate_every: Calculate error rate every N iterations and save + """ + logprog = lambda i, err, msg: logging.info("[{:6d} / {:2.2f}%] {:s}".format(i, (err) * 100, msg)) + self.ocr.prepare_training( + self.graphemes, + num_hidden=self.num_hidden, + learning_rate=self.learning_rate, + momentum=self.momentum) + + logging.info("Learning from {} [n-train={},n-test={}]".format( + self.imgdir, len(self.train_data), len(self.test_data))) + if save_every > -1: logging.info("Saving every {}th iteration".format(save_every)) + logging.info("Evaluating every {}th iteration".format(calculate_every)) + logging.info("Logging everyy {}th iteration".format(log_every)) + err = 1. + for i in range(self.max_iterations): + img, txt = random.choice(self.train_data) + out = self.ocr.train(img, txt) + if i == 0: continue + if not i % log_every: + aligned = self.ocr.aligned() + logprog(i, err, "TRN: {}".format(txt)) + logprog(i, err, "ALN: {}".format(aligned)) + if log_out: logprog(i, err, "OUT: {}".format(out)) + if not i % calculate_every: + cur_err = self.calculate_error() + if cur_err != err: + diff = 100*(err-cur_err) + logging.info("=== {}ed by {:.2f}%".format("Improv" if diff>0 else "Degrad", diff)) + self.save_model() + err = cur_err + else: + logging.info("No change in error rate") + elif save_every > 0 and not i % save_every: + self.save_model(suffix="{:05d}".format(i)) + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('imgdir', help='Directory matching "*/*.bin.png"') + parser.add_argument( + '--model-name', help='Name of the model file without ".clstm". Default: Generated from input name') + parser.add_argument( + '--gtdir', help='Directory matching "*/*.gt.txt". Default: imgdir') + parser.add_argument( + '--learning-rate', + metavar='RATE', + type=float, + default=0.0001, + help="Learning rate for the model training. Default: %(default)s") + parser.add_argument( + '--momentum', + type=float, + default=0.9, + help="Momentum for the model training. Default: %(default)s") + parser.add_argument( + '--num-hidden', + metavar='NUM', + type=int, + default=100, + help="Number of hidden units in the LSTM layers, larger " + "values require more storage/memory and take longer " + "for training and recognition, so try to find " + "a good performance/cost tradeoff. Default: %(default)s") + parser.add_argument( + '--max-iterations', + metavar='N', + type=int, + default=2000000, + help="Maximum iterations. Default: %(default)s") + parser.add_argument( + '--train-ratio', + metavar='RATIO', + type=float, + default=0.8, + help="Ratio of ground truth to be used for training. Default: %(default)s") + parser.add_argument( + '--log-out', + action='store_true', + default=False, + help="Log 'OUT' messages. Default: %(default)s") + parser.add_argument( + '--log-every', + type=int, + metavar='N', + default=10, + help="Log on every N-th iteration. Default: %(default)s") + parser.add_argument( + '--save-every', + type=int, + metavar='N', + default=500, + help="Save on every N-th iteration. Default: %(default)s") + parser.add_argument( + '--calculate-every', + type=int, + metavar='N', + default=1000, + help="Calculate error rate every N-th iteration. Default: %(default)s") + # imgdir = '/home/kb/build/github.com/tmbdev/clstm/book' + args = parser.parse_args() + trainer = OcrTrainer( + args.imgdir, + gtdir=args.gtdir, + model_name=args.model_name, + train_ratio=args.train_ratio, + max_iterations=args.max_iterations, + num_hidden=args.num_hidden, + learning_rate=args.learning_rate, + momentum=args.momentum) + trainer.train( + log_out=args.log_out, + log_every=args.log_every, + save_every=args.save_every, + calculate_every=args.calculate_every) + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index a6d2a4d..6adb75a 100644 --- a/setup.py +++ b/setup.py @@ -35,6 +35,12 @@ def ensure_protobuf(): setup( name='clstm', version='0.1', + packages = ['pyclstm_cli'], author="Thomas Breuel, Johannes Baiter", description="CLSTM Python bindings", + entry_points = { + 'console_scripts':[ + 'pyclstm-train=pyclstm_cli.train:main' + ] + }, ext_modules=cythonize([ext], compiler_directives={'embedsignature': True}))