From 560901d26793c6dd3a75069b8a46e3a6115a3503 Mon Sep 17 00:00:00 2001 From: meenchen Date: Thu, 16 Feb 2023 13:28:45 -0500 Subject: [PATCH 01/19] fuse reshape --- code_generator/TfliteConvertor.py | 5 ++++ .../converters/tflite_parser/__init__.py | 1 + .../converters/tflite_parser/reshape.py | 27 +++++++++++++++++++ 3 files changed, 33 insertions(+) create mode 100644 code_generator/converters/tflite_parser/reshape.py diff --git a/code_generator/TfliteConvertor.py b/code_generator/TfliteConvertor.py index 134d35ee..d9a93a11 100644 --- a/code_generator/TfliteConvertor.py +++ b/code_generator/TfliteConvertor.py @@ -17,9 +17,11 @@ # ---------------------------------------------------------------------- import logging +from typing import List import code_generator.converters.tflite_parser as TF_Parser from code_generator.converters.tflite_parser.mean1dto2d import MEAN2D +from code_generator.converters.tflite_parser.reshape import tensor_mapping from code_generator.converters.tflite_parser.utils import get_input_tensors, get_output_tensors, getOpCodeStr from .constant import SKIP_OPs @@ -37,6 +39,7 @@ def __init__(self, filepath): self.tmpPADIndice = None self.skip_transpose = None self.average_1D_to_2D_holder = MEAN2D() # For merging 1D to 2D + self.inplace_reshape_table: List[tensor_mapping] = [] # A list of tensor_mapping # public functions def loadTFmodel(self, filepath): @@ -139,6 +142,8 @@ def _handleOperator(self, op): self._convert_TRANSPOSE(op) elif op_code_str in "FULLY_CONNECTED": self.layer.append(TF_Parser.parse_fc(op, self.model)) + elif op_code_str in "RESHAPE": + self.inplace_reshape_table.append(TF_Parser.parse_reshape_fuse_tensor_tuple(op, self.model)) elif op_code_str in SKIP_OPs: pass else: diff --git a/code_generator/converters/tflite_parser/__init__.py b/code_generator/converters/tflite_parser/__init__.py index fefa6160..17378065 100644 --- a/code_generator/converters/tflite_parser/__init__.py +++ b/code_generator/converters/tflite_parser/__init__.py @@ -4,5 +4,6 @@ from .fc import parse_fc from .maxpool import parse_maxpool from .mean1dto2d import parse_mead1dto2d +from .reshape import parse_reshape_fuse_tensor_tuple from .SEelement import parse_SEelement from .upsample import parse_upsample diff --git a/code_generator/converters/tflite_parser/reshape.py b/code_generator/converters/tflite_parser/reshape.py new file mode 100644 index 00000000..f2a64c90 --- /dev/null +++ b/code_generator/converters/tflite_parser/reshape.py @@ -0,0 +1,27 @@ +from code_generator.tflite import Model + +from .utils import get_input_tensors, get_output_tensors, getTensorTypeStr + + +class tensor_mapping: + input_idx: str + output_idx: str + + def __init__(self, input_idx, output_idx) -> None: + self.input_idx = input_idx + self.output_idx = output_idx + + +def parse_reshape_fuse_tensor_tuple(op, model: Model.Model): + # get input, weight, and output tensors + input_tensors = get_input_tensors(op, model) + + output_tensors = get_output_tensors(op, model) + output_tensor = output_tensors[0] + + # tensor types + input_type = getTensorTypeStr(input_tensors[0].tensor.Type()) + output_type = getTensorTypeStr(output_tensor.tensor.Type()) + assert input_type == output_type, "tensor type not consistent" + + return tensor_mapping(input_tensors[0].tensor_idx, output_tensor.tensor_idx) From 7fd4c6c6b2c7f68d07a24fe09ca293f98d521ac7 Mon Sep 17 00:00:00 2001 From: meenchen Date: Thu, 16 Feb 2023 18:28:24 -0500 Subject: [PATCH 02/19] op entry points --- code_generator/TfliteConvertor.py | 33 +++++- .../converters/tflite_parser/__init__.py | 11 ++ .../converters/tflite_parser/add.py | 36 ++++++- .../converters/tflite_parser/batchmatmul.py | 56 ++++++++++ .../converters/tflite_parser/cast.py | 43 ++++++++ .../converters/tflite_parser/concat.py | 100 ++++++++++++++++++ .../converters/tflite_parser/div.py | 97 +++++++++++++++++ .../converters/tflite_parser/equal.py | 55 ++++++++++ .../converters/tflite_parser/mul.py | 65 ++++++++++++ .../converters/tflite_parser/notequal.py | 55 ++++++++++ .../converters/tflite_parser/rsqrt.py | 43 ++++++++ .../converters/tflite_parser/slice.py | 54 ++++++++++ .../converters/tflite_parser/softmax.py | 50 +++++++++ .../converters/tflite_parser/squarddiff.py | 72 +++++++++++++ .../converters/tflite_parser/sub.py | 72 +++++++++++++ code_generator/operators/__init__.py | 7 ++ code_generator/operators/add.py | 23 ++-- code_generator/operators/basic_utils.py | 14 +-- code_generator/operators/batchmatmul.py | 71 +++++++++++++ code_generator/operators/cast.py | 5 +- code_generator/operators/concat.py | 74 +++++++++++++ code_generator/operators/div.py | 12 ++- code_generator/operators/equal.py | 71 +++++++++++++ code_generator/operators/mul.py | 2 +- code_generator/operators/notequal.py | 71 +++++++++++++ code_generator/operators/rsqrt.py | 52 +++++++++ code_generator/operators/slice.py | 59 +++++++++++ code_generator/operators/softmax.py | 81 ++++++++++++++ code_generator/operators/squarddiff.py | 75 +++++++++++++ code_generator/operators/sub.py | 6 ++ 30 files changed, 1438 insertions(+), 27 deletions(-) create mode 100644 code_generator/converters/tflite_parser/batchmatmul.py create mode 100644 code_generator/converters/tflite_parser/cast.py create mode 100644 code_generator/converters/tflite_parser/concat.py create mode 100644 code_generator/converters/tflite_parser/div.py create mode 100644 code_generator/converters/tflite_parser/equal.py create mode 100644 code_generator/converters/tflite_parser/mul.py create mode 100644 code_generator/converters/tflite_parser/notequal.py create mode 100644 code_generator/converters/tflite_parser/rsqrt.py create mode 100644 code_generator/converters/tflite_parser/slice.py create mode 100644 code_generator/converters/tflite_parser/softmax.py create mode 100644 code_generator/converters/tflite_parser/squarddiff.py create mode 100644 code_generator/converters/tflite_parser/sub.py create mode 100644 code_generator/operators/batchmatmul.py create mode 100644 code_generator/operators/concat.py create mode 100644 code_generator/operators/equal.py create mode 100644 code_generator/operators/notequal.py create mode 100644 code_generator/operators/rsqrt.py create mode 100644 code_generator/operators/slice.py create mode 100644 code_generator/operators/softmax.py create mode 100644 code_generator/operators/squarddiff.py diff --git a/code_generator/TfliteConvertor.py b/code_generator/TfliteConvertor.py index d9a93a11..b9084730 100644 --- a/code_generator/TfliteConvertor.py +++ b/code_generator/TfliteConvertor.py @@ -111,6 +111,9 @@ def parseOperatorInfo(self): # parse the op self._handleOperator(op) + # Handle inplace_reshape_table here + logging.error("Please handle inplace_reshape_table here for fused tensors.") + # handle one op and parse it into layers[] for supported operators def _handleOperator(self, op): op_code_str = getOpCodeStr(op, self.model) @@ -140,11 +143,35 @@ def _handleOperator(self, op): self.layer.append(ret_op) elif op_code_str == "TRANSPOSE": self._convert_TRANSPOSE(op) - elif op_code_str in "FULLY_CONNECTED": + elif op_code_str == "FULLY_CONNECTED": self.layer.append(TF_Parser.parse_fc(op, self.model)) - elif op_code_str in "RESHAPE": + elif op_code_str == "RESHAPE": self.inplace_reshape_table.append(TF_Parser.parse_reshape_fuse_tensor_tuple(op, self.model)) - elif op_code_str in SKIP_OPs: + elif op_code_str == "DIV": + self.layer.append(TF_Parser.parse_div(op, self.model)) + elif op_code_str == "BATCH_MATMUL": + self.layer.append(TF_Parser.parse_batchmatmul(op, self.model)) + elif op_code_str == "NOT_EQUAL": + self.layer.append(TF_Parser.parse_notequal(op, self.model)) + elif op_code_str == "EQUAL": + self.layer.append(TF_Parser.parse_equal(op, self.model)) + elif op_code_str == "CONCATENATION": + self.layer.append(TF_Parser.parse_notequal(op, self.model)) + elif op_code_str == "CAST": + self.layer.append(TF_Parser.parse_cast(op, self.model)) + elif op_code_str == "SUB": + self.layer.append(TF_Parser.parse_sub(op, self.model)) + elif op_code_str == "MUL": + self.layer.append(TF_Parser.parse_mul(op, self.model)) + elif op_code_str == "SOFTMAX": + self.layer.append(TF_Parser.parse_softmax(op, self.model)) + elif op_code_str == "SQUARED_DIFFERENCE": + self.layer.append(TF_Parser.parse_squarddiff(op, self.model)) + elif op_code_str == "RSQRT": + self.layer.append(TF_Parser.parse_rsqrt(op, self.model)) + elif op_code_str == "SLICE": + self.layer.append(TF_Parser.parse_slice(op, self.model)) + elif op_code_str == SKIP_OPs: pass else: raise NotImplementedError(f"Unsupported {op_code_str}") diff --git a/code_generator/converters/tflite_parser/__init__.py b/code_generator/converters/tflite_parser/__init__.py index 17378065..1c915f17 100644 --- a/code_generator/converters/tflite_parser/__init__.py +++ b/code_generator/converters/tflite_parser/__init__.py @@ -1,9 +1,20 @@ from .add import parse_add from .avgpool import parse_avgpool +from .batchmatmul import parse_batchmatmul +from .cast import parse_cast from .conv2d import parse_conv2d +from .div import parse_div +from .equal import parse_equal from .fc import parse_fc from .maxpool import parse_maxpool from .mean1dto2d import parse_mead1dto2d +from .mul import parse_mul +from .notequal import parse_notequal from .reshape import parse_reshape_fuse_tensor_tuple +from .rsqrt import parse_rsqrt from .SEelement import parse_SEelement +from .slice import parse_slice +from .softmax import parse_softmax +from .squarddiff import parse_squarddiff +from .sub import parse_sub from .upsample import parse_upsample diff --git a/code_generator/converters/tflite_parser/add.py b/code_generator/converters/tflite_parser/add.py index 07cc60e8..33ea5f3e 100644 --- a/code_generator/converters/tflite_parser/add.py +++ b/code_generator/converters/tflite_parser/add.py @@ -5,7 +5,14 @@ from code_generator.operators import add from code_generator.tflite import Model -from .utils import get_input_tensors, get_nhwc_from_shape, get_output_tensors, getOpCodeStr, getTensorTypeStr +from .utils import ( + get_input_tensors, + get_nhwc_from_shape, + get_np_from_wrapper, + get_output_tensors, + getOpCodeStr, + getTensorTypeStr, +) def parse_add(op, model: Model.Model): @@ -28,9 +35,19 @@ def parse_add(op, model: Model.Model): _, input_h, input_w, input_c = get_nhwc_from_shape(input_tensor.tensor.ShapeAsNumpy()) _, input2_h, input2_w, input2_c = get_nhwc_from_shape(input2_tensor.tensor.ShapeAsNumpy()) _, output_h, output_w, output_c = get_nhwc_from_shape(output_tensor.tensor.ShapeAsNumpy()) - assert input_h == input2_h == output_h, "tensor shpae not consistent" - assert input_w == input2_w == output_w, "tensor shpae not consistent" - assert input_c == input2_c == output_c, "tensor shpae not consistent" + + # Find out which tensor is the main input, we assume the 1st input is the main one + if input2_c == output_c and output_w == input2_w and output_h == input2_h: + temp = input_tensor + input2_tensor = input_tensor + input_tensor = temp + th, tw, tc = input_h, input_w, input_c + input_h, input_w, input_c = input2_h, input2_w, input2_c + input2_h, input2_w, input2_c = th, tw, tc + + assert input_h == output_h, "tensor shape not consistent" + assert input_w == output_w, "tensor shape not consistent" + assert input_c == output_c, "tensor shape not consistent" # tensor types input_type = getTensorTypeStr(input_tensor.tensor.Type()) @@ -38,6 +55,14 @@ def parse_add(op, model: Model.Model): output_type = getTensorTypeStr(output_tensor.tensor.Type()) assert input_type == input_type2 == output_type, "tensor type not consistent" + # Check if the divisor is constant, e.g., a scale value or tensor + input2_idx = input2_tensor.tensor_idx + try: + input2 = get_np_from_wrapper(input2_tensor) + input2_idx = "constant" + except Exception: + input2 = None + # initialize quantized parameters as None for floating-pointer ops input_zero_point = None input_scale = None @@ -83,7 +108,8 @@ def parse_add(op, model: Model.Model): "op": op_code_str, # tensor "input_idx": input_tensor.tensor_idx, - "input2_idx": input2_tensor.tensor_idx, + "input2": input2, + "input2_idx": input2_idx, "output_idx": output_tensor.tensor_idx, "input_h": input_h, "input_w": input_w, diff --git a/code_generator/converters/tflite_parser/batchmatmul.py b/code_generator/converters/tflite_parser/batchmatmul.py new file mode 100644 index 00000000..152ed312 --- /dev/null +++ b/code_generator/converters/tflite_parser/batchmatmul.py @@ -0,0 +1,56 @@ +from code_generator.operators import batchmatmul +from code_generator.tflite import Model + +from .utils import get_input_tensors, get_nhwc_from_shape, get_output_tensors, getTensorTypeStr + + +def parse_batchmatmul(op, model: Model.Model): + # get input, weight, and output tensors + input_tensors = get_input_tensors(op, model) + input_tensor_count = len(input_tensors) + assert input_tensor_count == 2, "input tensors length should be 2" + + input_tensor = input_tensors[0] + input2_tensor = input_tensors[1] + + output_tensors = get_output_tensors(op, model) + assert len(output_tensors) == 1, "output tensors length should be 1" + output_tensor = output_tensors[0] + + # shapes + _, i_batch_size, input_row, input_col = get_nhwc_from_shape(input_tensor.tensor.ShapeAsNumpy()) + _, i2_batch_size, input2_row, input2_col = get_nhwc_from_shape(input2_tensor.tensor.ShapeAsNumpy()) + _, o_batch_size, output_row, output_col = get_nhwc_from_shape(output_tensor.tensor.ShapeAsNumpy()) + assert i_batch_size == i2_batch_size == o_batch_size, "batch size not match" + assert input_col == input2_row, "matric dimension not match" + assert input2_col == output_col, "matric dimension not match" + assert input_row == output_row, "matric dimension not match" + + # tensor types + input_type = getTensorTypeStr(input_tensor.tensor.Type()) + input2_type = getTensorTypeStr(input2_tensor.tensor.Type()) + output_type = getTensorTypeStr(output_tensor.tensor.Type()) + assert input_type == output_type == input2_type, "tensor type not consistent" + + params = { + # op related + "op": "BATCH_MATMUL", + "batch_size": i_batch_size, + "input_idx": input_tensor.tensor_idx, + "input2_idx": input2_tensor.tensor_idx, + "output_idx": output_tensor.tensor_idx, + # tensor related + "input_col": input_col, + "input_row": input_row, + "input2": None, + "input2_col": input2_col, + "input2_row": input2_row, + "output_col": output_col, + "output_row": output_row, + "input_dtype": "fp32", + "input2_dtype": "fp32", + "output_dtype": "fp32", + } + op = batchmatmul.batchmatmul(params) + + return op diff --git a/code_generator/converters/tflite_parser/cast.py b/code_generator/converters/tflite_parser/cast.py new file mode 100644 index 00000000..3dd45de6 --- /dev/null +++ b/code_generator/converters/tflite_parser/cast.py @@ -0,0 +1,43 @@ +from code_generator.operators import cast +from code_generator.tflite import Model + +from .utils import get_input_tensors, get_nhwc_from_shape, get_output_tensors, getTensorTypeStr + + +def parse_cast(op, model: Model.Model): + # get input, weight, and output tensors + input_tensors = get_input_tensors(op, model) + input_tensor = input_tensors[0] + + output_tensors = get_output_tensors(op, model) + assert len(output_tensors) == 1, "output tensors length should be 1" + output_tensor = output_tensors[0] + + # shapes + _, input_h, input_w, input_c = get_nhwc_from_shape(input_tensor.tensor.ShapeAsNumpy()) + _, output_h, output_w, output_c = get_nhwc_from_shape(output_tensor.tensor.ShapeAsNumpy()) + + assert input_h == output_h, "tensor shape not consistent" + assert input_w == output_w, "tensor shape not consistent" + assert input_c == output_c, "tensor shape not consistent" + + # tensor types + input_type = getTensorTypeStr(input_tensor.tensor.Type()) + output_type = getTensorTypeStr(output_tensor.tensor.Type()) + assert input_type != output_type, "expect tensor types to be inconsistent" + + # assign params + params = { + # op related + "op": "CAST", + "input_idx": input_tensor.tensor_idx, + "output_idx": output_tensor.tensor_idx, + # tensor related + "input_size": input_h * input_w * input_c, + "input_dtype": input_type, + "output_dtype": output_type, + "input_meta": None, + } + op = cast.cast(params) + + return op diff --git a/code_generator/converters/tflite_parser/concat.py b/code_generator/converters/tflite_parser/concat.py new file mode 100644 index 00000000..726b3ad7 --- /dev/null +++ b/code_generator/converters/tflite_parser/concat.py @@ -0,0 +1,100 @@ +from code_generator.operators import concat +from code_generator.tflite import Model + +from .utils import ( + get_input_tensors, + get_nhwc_from_shape, + get_np_from_wrapper, + get_output_tensors, + getOpCodeStr, + getTensorTypeStr, +) + + +def parse_concat(op, model: Model.Model): + # operator + op_code_str = getOpCodeStr(op, model) + + # get input, weight, and output tensors + input_tensors = get_input_tensors(op, model) + input_tensor_count = len(input_tensors) + assert input_tensor_count == 2, "input should be 2 tensors" + + input_tensor = input_tensors[0] + input2_tensor = input_tensors[1] + + output_tensors = get_output_tensors(op, model) + assert len(output_tensors) == 1, "output tensors length should be 1" + output_tensor = output_tensors[0] + + # shapes + _, input_h, input_w, input_c = get_nhwc_from_shape(input_tensor.tensor.ShapeAsNumpy()) + _, input2_h, input2_w, input2_c = get_nhwc_from_shape(input2_tensor.tensor.ShapeAsNumpy()) + _, output_h, output_w, output_c = get_nhwc_from_shape(output_tensor.tensor.ShapeAsNumpy()) + + assert input_h == output_h, "tensor shape not consistent" + assert input_w == output_w, "tensor shape not consistent" + assert input_c == output_c, "tensor shape not consistent" + + # tensor types + input_type = getTensorTypeStr(input_tensor.tensor.Type()) + input_type2 = getTensorTypeStr(input2_tensor.tensor.Type()) + output_type = getTensorTypeStr(output_tensor.tensor.Type()) + assert input_type == input_type2 == output_type, "tensor type not consistent" + + # Check if the divisor is constant, e.g., a scale value or tensor + input2_idx = input2_tensor.tensor_idx + try: + input2 = get_np_from_wrapper(input2_tensor) + input2_idx = "constant" + except Exception: + input2 = None + + # initialize quantized parameters as None for floating-pointer ops + input_zero_point = None + input_scale = None + input2_zero_point = None + input2_scale = None + output_zero_point = None + output_scale = None + + if "float32" not in [output_type, input_type, input_type2]: + raise NotImplementedError("only support floating point for now.") + + # assign params + params = { + # operator + "op": op_code_str, + # tensor + "input_idx": input_tensor.tensor_idx, + "input_size": input_h * input_w * input_c, + "input2_idx": input2_idx, + "input2": input2, + "output_idx": output_tensor.tensor_idx, + "input_h": input_h, + "input_w": input_w, + "input_c": input_c, + "input2_h": input2_h, + "input2_w": input2_w, + "input2_c": input2_c, + "input_dim": 3, + "input2_dim": 3, + "output_dim": 3, + "output_h": output_h, + "output_w": output_w, + "output_c": output_c, + "dtypte": input_type, + "input_dtype": input_type, + "input2_dtype": input_type2, + "output_dtype": output_type, + # trainable parameters + "input_zero_point": input_zero_point, + "input2_zero_point": input2_zero_point, + "output_zero_point": output_zero_point, + "input_scale": input_scale, + "input2_scale": input2_scale, + "output_scale": output_scale, + } + op = concat.concat(params) + + return op diff --git a/code_generator/converters/tflite_parser/div.py b/code_generator/converters/tflite_parser/div.py new file mode 100644 index 00000000..795452c2 --- /dev/null +++ b/code_generator/converters/tflite_parser/div.py @@ -0,0 +1,97 @@ +from code_generator.operators import div +from code_generator.tflite import Model + +from .utils import ( + get_input_tensors, + get_nhwc_from_shape, + get_np_from_wrapper, + get_output_tensors, + getOpCodeStr, + getTensorTypeStr, +) + + +def parse_div(op, model: Model.Model): + # operator + op_code_str = getOpCodeStr(op, model) + + # get input, weight, and output tensors + input_tensors = get_input_tensors(op, model) + input_tensor_count = len(input_tensors) + assert input_tensor_count == 2, "input should be 2 tensors" + + input_tensor = input_tensors[0] + input2_tensor = input_tensors[1] + + output_tensors = get_output_tensors(op, model) + assert len(output_tensors) == 1, "output tensors length should be 1" + output_tensor = output_tensors[0] + + # shapes + _, input_h, input_w, input_c = get_nhwc_from_shape(input_tensor.tensor.ShapeAsNumpy()) + _, input2_h, input2_w, input2_c = get_nhwc_from_shape(input2_tensor.tensor.ShapeAsNumpy()) + _, output_h, output_w, output_c = get_nhwc_from_shape(output_tensor.tensor.ShapeAsNumpy()) + + assert input_h == output_h, "tensor shpae not consistent" + assert input_w == output_w, "tensor shpae not consistent" + assert input_c == output_c, "tensor shpae not consistent" + + # tensor types + input_type = getTensorTypeStr(input_tensor.tensor.Type()) + input_type2 = getTensorTypeStr(input2_tensor.tensor.Type()) + output_type = getTensorTypeStr(output_tensor.tensor.Type()) + assert input_type == input_type2 == output_type, "tensor type not consistent" + + # Check if the divisor is constant, e.g., a scale value or tensor + input2_idx = input2_tensor.tensor_idx + try: + input2 = get_np_from_wrapper(input2_tensor) + input2_idx = "constant" + except Exception: + input2 = None + + # initialize quantized parameters as None for floating-pointer ops + input_zero_point = None + input_scale = None + input2_zero_point = None + input2_scale = None + output_zero_point = None + output_scale = None + + if "float32" not in [output_type, input_type, input_type2]: + raise NotImplementedError("only support floating point for now.") + + # assign params + params = { + # operator + "op": op_code_str, + # tensor + "input_idx": input_tensor.tensor_idx, + "input_size": input_h * input_w * input_c, + "input2_idx": input2_idx, + "input2": input2, + "output_idx": output_tensor.tensor_idx, + "input_h": input_h, + "input_w": input_w, + "input_c": input_c, + "input2_h": input2_h, + "input2_w": input2_w, + "input2_c": input2_c, + "output_h": output_h, + "output_w": output_w, + "output_c": output_c, + "dtypte": input_type, + "input_dtype": input_type, + "input2_dtype": input_type2, + "output_dtype": output_type, + # trainable parameters + "input_zero_point": input_zero_point, + "input2_zero_point": input2_zero_point, + "output_zero_point": output_zero_point, + "input_scale": input_scale, + "input2_scale": input2_scale, + "output_scale": output_scale, + } + op = div.div(params) + + return op diff --git a/code_generator/converters/tflite_parser/equal.py b/code_generator/converters/tflite_parser/equal.py new file mode 100644 index 00000000..8fb1493d --- /dev/null +++ b/code_generator/converters/tflite_parser/equal.py @@ -0,0 +1,55 @@ +from code_generator.operators import equal +from code_generator.tflite import Model + +from .utils import get_input_tensors, get_nhwc_from_shape, get_np_from_wrapper, get_output_tensors, getTensorTypeStr + + +def parse_equal(op, model: Model.Model): + # get input, weight, and output tensors + input_tensors = get_input_tensors(op, model) + + output_tensors = get_output_tensors(op, model) + output_tensor = output_tensors[0] + + # tensor types + input_type = getTensorTypeStr(input_tensors[0].tensor.Type()) + input2_type = getTensorTypeStr(input_tensors[1].tensor.Type()) + output_type = getTensorTypeStr(output_tensor.tensor.Type()) + assert input_type == output_type == input2_type, "tensor type not consistent" + + # shapes + _, input_h, input_w, input_c = get_nhwc_from_shape(input_tensors[0].tensor.ShapeAsNumpy()) + _, input2_h, input2_w, input2_c = get_nhwc_from_shape(input_tensors[1].tensor.ShapeAsNumpy()) + _, output_h, output_w, output_c = get_nhwc_from_shape(output_tensor.tensor.ShapeAsNumpy()) + + # Check if the divisor is constant, e.g., a scale value or tensor + input2_idx = input_tensors[1].tensor_idx + try: + input2 = get_np_from_wrapper(input_tensors[1]) + input2_idx = "constant" + except Exception: + input2 = None + + params = { + # op related + "op": "EQUAL", + "input_idx": input_tensors[0].tensor_idx, + "input2_idx": input2_idx, + "output_idx": output_tensors[0].tensor_idx, + # tensor related + "input_h": input_h, + "input_w": input_w, + "input_c": input_c, + "input2": input2, + "input2_h": input2_h, + "input2_w": input2_w, + "input2_c": input2_c, + "output_h": output_h, + "output_w": output_w, + "output_c": output_c, + "input_dtype": input_type, + "input2_dtype": input2_type, + "output_dtype": output_type, + } + + return equal.equal(params) diff --git a/code_generator/converters/tflite_parser/mul.py b/code_generator/converters/tflite_parser/mul.py new file mode 100644 index 00000000..18282761 --- /dev/null +++ b/code_generator/converters/tflite_parser/mul.py @@ -0,0 +1,65 @@ +from code_generator.operators import mul +from code_generator.tflite import Model + +from .utils import get_input_tensors, get_nhwc_from_shape, get_np_from_wrapper, get_output_tensors, getTensorTypeStr + + +def parse_mul(op, model: Model.Model): + # get input, weight, and output tensors + input_tensors = get_input_tensors(op, model) + input_tensor = input_tensors[0] + input2_tensor = input_tensors[1] + + output_tensors = get_output_tensors(op, model) + output_tensor = output_tensors[0] + + # shapes + _, input_h, input_w, input_c = get_nhwc_from_shape(input_tensor.tensor.ShapeAsNumpy()) + _, input2_h, input2_w, input2_c = get_nhwc_from_shape(input2_tensor.tensor.ShapeAsNumpy()) + _, output_h, output_w, output_c = get_nhwc_from_shape(output_tensor.tensor.ShapeAsNumpy()) + + # Find out which tensor is the main input, we assume the 1st input is the main one + if input2_c == output_c and output_w == input2_w and output_h == input2_h: + temp = input_tensor + input2_tensor = input_tensor + input_tensor = temp + th, tw, tc = input_h, input_w, input_c + input_h, input_w, input_c = input2_h, input2_w, input2_c + input2_h, input2_w, input2_c = th, tw, tc + + assert input_h == output_h, "tensor shape not consistent" + assert input_w == output_w, "tensor shape not consistent" + assert input_c == output_c, "tensor shape not consistent" + + # tensor types + input_type = getTensorTypeStr(input_tensor.tensor.Type()) + output_type = getTensorTypeStr(output_tensor.tensor.Type()) + assert input_type == output_type, "tensor type is not consistent" + + # Check if the divisor is constant, e.g., a scale value or tensor + input2_idx = input2_tensor.tensor_idx + try: + input2 = get_np_from_wrapper(input2_tensor) + input2_idx = "constant" + except Exception: + input2 = None + + # assign params + params = { + # op related + "op": "MUL", + "input_idx": input_tensor.tensor_idx, + "input2": input2, + "input2_idx": input2_idx, + "output_idx": output_tensor.tensor_idx, + # tensor related + "input_size": input_h * input_w * input_c, + "input2_size": input2_h * input2_w * input2_c, + "output_size": output_h * output_w * output_c, + "input_dtype": input_type, + "output_dtype": output_type, + "input_meta": None, + } + op = mul.mul(params) + + return op diff --git a/code_generator/converters/tflite_parser/notequal.py b/code_generator/converters/tflite_parser/notequal.py new file mode 100644 index 00000000..c44c4985 --- /dev/null +++ b/code_generator/converters/tflite_parser/notequal.py @@ -0,0 +1,55 @@ +from code_generator.operators import notequal +from code_generator.tflite import Model + +from .utils import get_input_tensors, get_nhwc_from_shape, get_np_from_wrapper, get_output_tensors, getTensorTypeStr + + +def parse_notequal(op, model: Model.Model): + # get input, weight, and output tensors + input_tensors = get_input_tensors(op, model) + + output_tensors = get_output_tensors(op, model) + output_tensor = output_tensors[0] + + # tensor types + input_type = getTensorTypeStr(input_tensors[0].tensor.Type()) + input2_type = getTensorTypeStr(input_tensors[1].tensor.Type()) + output_type = getTensorTypeStr(output_tensor.tensor.Type()) + assert input_type == output_type == input2_type, "tensor type not consistent" + + # shapes + _, input_h, input_w, input_c = get_nhwc_from_shape(input_tensors[0].tensor.ShapeAsNumpy()) + _, input2_h, input2_w, input2_c = get_nhwc_from_shape(input_tensors[1].tensor.ShapeAsNumpy()) + _, output_h, output_w, output_c = get_nhwc_from_shape(output_tensor.tensor.ShapeAsNumpy()) + + # Check if the divisor is constant, e.g., a scale value or tensor + input2_idx = input_tensors[1].tensor_idx + try: + input2 = get_np_from_wrapper(input_tensors[1]) + input2_idx = "constant" + except Exception: + input2 = None + + params = { + # op related + "op": "NOT_EQUAL", + "input_idx": input_tensors[0].tensor_idx, + "input2_idx": input2_idx, + "output_idx": output_tensors[0].tensor_idx, + # tensor related + "input_h": input_h, + "input_w": input_w, + "input_c": input_c, + "input2": input2, + "input2_h": input2_h, + "input2_w": input2_w, + "input2_c": input2_c, + "output_h": output_h, + "output_w": output_w, + "output_c": output_c, + "input_dtype": input_type, + "input2_dtype": input2_type, + "output_dtype": output_type, + } + + return notequal.notequal(params) diff --git a/code_generator/converters/tflite_parser/rsqrt.py b/code_generator/converters/tflite_parser/rsqrt.py new file mode 100644 index 00000000..f667b87b --- /dev/null +++ b/code_generator/converters/tflite_parser/rsqrt.py @@ -0,0 +1,43 @@ +from code_generator.operators import rsqrt +from code_generator.tflite import Model + +from .utils import get_input_tensors, get_nhwc_from_shape, get_output_tensors, getTensorTypeStr + + +def parse_rsqrt(op, model: Model.Model): + # get input, weight, and output tensors + input_tensors = get_input_tensors(op, model) + input_tensor = input_tensors[0] + + output_tensors = get_output_tensors(op, model) + assert len(output_tensors) == 1, "output tensors length should be 1" + output_tensor = output_tensors[0] + + # shapes + _, input_h, input_w, input_c = get_nhwc_from_shape(input_tensor.tensor.ShapeAsNumpy()) + _, output_h, output_w, output_c = get_nhwc_from_shape(output_tensor.tensor.ShapeAsNumpy()) + + assert input_h == output_h, "tensor shape not consistent" + assert input_w == output_w, "tensor shape not consistent" + assert input_c == output_c, "tensor shape not consistent" + + # tensor types + input_type = getTensorTypeStr(input_tensor.tensor.Type()) + output_type = getTensorTypeStr(output_tensor.tensor.Type()) + assert input_type == output_type, "tensor types inconsistent" + + # assign params + params = { + # op related + "op": "RSQRT", + "input_idx": input_tensor.tensor_idx, + "output_idx": output_tensor.tensor_idx, + # tensor related + "input_size": input_h * input_w * input_c, + "input_dtype": input_type, + "output_dtype": output_type, + "input_meta": None, + } + op = rsqrt.rsqrt(params) + + return op diff --git a/code_generator/converters/tflite_parser/slice.py b/code_generator/converters/tflite_parser/slice.py new file mode 100644 index 00000000..53ecd410 --- /dev/null +++ b/code_generator/converters/tflite_parser/slice.py @@ -0,0 +1,54 @@ +from code_generator.operators import slice +from code_generator.tflite import Model + +from .utils import get_input_tensors, get_nhwc_from_shape, get_output_tensors, getTensorTypeStr + + +def parse_slice(op, model: Model.Model): + # get input, weight, and output tensors + input_tensors = get_input_tensors(op, model) + assert len(input_tensors) == 3, "intput tensors length should be 3" + input_tensor = input_tensors[0] + begin_tensor = input_tensors[1] + end_tensor = input_tensors[1] + + output_tensors = get_output_tensors(op, model) + assert len(output_tensors) == 1, "output tensors length should be 1" + output_tensor = output_tensors[0] + + # shapes + _, input_h, input_w, input_c = get_nhwc_from_shape(input_tensor.tensor.ShapeAsNumpy()) + _, output_h, output_w, output_c = get_nhwc_from_shape(output_tensor.tensor.ShapeAsNumpy()) + _, begin_h, begin_w, begin_c = get_nhwc_from_shape(begin_tensor.tensor.ShapeAsNumpy()) + _, end_h, end_w, end_c = get_nhwc_from_shape(end_tensor.tensor.ShapeAsNumpy()) + + # tensor types + input_type = getTensorTypeStr(input_tensor.tensor.Type()) + output_type = getTensorTypeStr(output_tensor.tensor.Type()) + assert input_type == output_type, "tensor types inconsistent" + + # assign params + params = { + # op related + "op": "SLICE", + "input_idx": input_tensor.tensor_idx, + "output_idx": output_tensor.tensor_idx, + # tensor related + "input_h": input_h, + "input_w": input_w, + "input_c": input_c, + "begin_h": begin_h, + "begin_w": begin_w, + "begin_c": begin_c, + "end_h": end_h, + "end_w": end_w, + "end_c": end_c, + "output_h": output_h, + "output_w": output_w, + "output_c": output_c, + "input_dtype": input_type, + "output_dtype": output_type, + } + op = slice.slice(params) + + return op diff --git a/code_generator/converters/tflite_parser/softmax.py b/code_generator/converters/tflite_parser/softmax.py new file mode 100644 index 00000000..5737a0d0 --- /dev/null +++ b/code_generator/converters/tflite_parser/softmax.py @@ -0,0 +1,50 @@ +from code_generator.operators import softmax +from code_generator.tflite import Model + +from .utils import get_input_tensors, get_nhwc_from_shape, get_output_tensors, getTensorTypeStr + + +def parse_softmax(op, model: Model.Model): + # get input, weight, and output tensors + input_tensors = get_input_tensors(op, model) + input_tensor = input_tensors[0] + assert len(input_tensors) == 1, "input tensors length should be 1" + + output_tensors = get_output_tensors(op, model) + assert len(output_tensors) == 1, "output tensors length should be 1" + output_tensor = output_tensors[0] + + # shapes + _, input_h, input_w, input_c = get_nhwc_from_shape(input_tensor.tensor.ShapeAsNumpy()) + _, output_h, output_w, output_c = get_nhwc_from_shape(output_tensor.tensor.ShapeAsNumpy()) + + assert input_h == output_h, "tensor shape not consistent" + assert input_w == output_w, "tensor shape not consistent" + assert input_c == output_c, "tensor shape not consistent" + + # tensor types + input_type = getTensorTypeStr(input_tensor.tensor.Type()) + output_type = getTensorTypeStr(output_tensor.tensor.Type()) + assert input_type == output_type, "tensor types inconsistent" + + # assign params + params = { + # op related + "op": "SOFTMAX", + "input_idx": input_tensor.tensor_idx, + "output_idx": output_tensor.tensor_idx, + "input_h": input_h, + "input_w": input_w, + "input_c": input_c, + "output_h": output_h, + "output_w": output_w, + "output_c": output_c, + # tensor related + "input_size": input_h * input_w * input_c, + "input_dtype": input_type, + "output_dtype": output_type, + "input_meta": None, + } + op = softmax.softmax(params) + + return op diff --git a/code_generator/converters/tflite_parser/squarddiff.py b/code_generator/converters/tflite_parser/squarddiff.py new file mode 100644 index 00000000..dbf87976 --- /dev/null +++ b/code_generator/converters/tflite_parser/squarddiff.py @@ -0,0 +1,72 @@ +from code_generator.operators import squarddiff +from code_generator.tflite import Model + +from .utils import get_input_tensors, get_nhwc_from_shape, get_np_from_wrapper, get_output_tensors, getTensorTypeStr + + +def parse_squarddiff(op, model: Model.Model): + # get input, weight, and output tensors + input_tensors = get_input_tensors(op, model) + input_tensor = input_tensors[0] + input2_tensor = input_tensors[1] + + output_tensors = get_output_tensors(op, model) + output_tensor = output_tensors[0] + + # shapes + _, input_h, input_w, input_c = get_nhwc_from_shape(input_tensor.tensor.ShapeAsNumpy()) + _, input2_h, input2_w, input2_c = get_nhwc_from_shape(input2_tensor.tensor.ShapeAsNumpy()) + _, output_h, output_w, output_c = get_nhwc_from_shape(output_tensor.tensor.ShapeAsNumpy()) + + # Find out which tensor is the main input, we assume the 1st input is the main one + if input2_c == output_c and output_w == input2_w and output_h == input2_h: + temp = input_tensor + input2_tensor = input_tensor + input_tensor = temp + th, tw, tc = input_h, input_w, input_c + input_h, input_w, input_c = input2_h, input2_w, input2_c + input2_h, input2_w, input2_c = th, tw, tc + + assert input_h == output_h, "tensor shape not consistent" + assert input_w == output_w, "tensor shape not consistent" + assert input_c == output_c, "tensor shape not consistent" + + # tensor types + input_type = getTensorTypeStr(input_tensor.tensor.Type()) + output_type = getTensorTypeStr(output_tensor.tensor.Type()) + assert input_type == output_type, "tensor type is not consistent" + + # Check if the divisor is constant, e.g., a scale value or tensor + input2_idx = input2_tensor.tensor_idx + try: + input2 = get_np_from_wrapper(input2_tensor) + input2_idx = "constant" + except Exception: + input2 = None + + # assign params + params = { + # op related + "op": "SQUARED_DIFFERENCE", + "input_idx": input_tensor.tensor_idx, + "input2": input2, + "input2_idx": input2_idx, + "output_idx": output_tensor.tensor_idx, + # tensor related + "input_h": input_h, + "input_w": input_w, + "input_c": input_c, + "input2_h": input2_h, + "input2_w": input2_w, + "input2_c": input2_c, + "output_h": output_h, + "output_w": output_w, + "output_c": output_c, + "input_size": input_h * input_w * input_c, + "input_dtype": input_type, + "output_dtype": output_type, + "input_meta": None, + } + op = squarddiff.squarddiff(params) + + return op diff --git a/code_generator/converters/tflite_parser/sub.py b/code_generator/converters/tflite_parser/sub.py new file mode 100644 index 00000000..a109f99d --- /dev/null +++ b/code_generator/converters/tflite_parser/sub.py @@ -0,0 +1,72 @@ +from code_generator.operators import sub +from code_generator.tflite import Model + +from .utils import get_input_tensors, get_nhwc_from_shape, get_np_from_wrapper, get_output_tensors, getTensorTypeStr + + +def parse_sub(op, model: Model.Model): + # get input, weight, and output tensors + input_tensors = get_input_tensors(op, model) + input_tensor = input_tensors[0] + input2_tensor = input_tensors[1] + + output_tensors = get_output_tensors(op, model) + output_tensor = output_tensors[0] + + # shapes + _, input_h, input_w, input_c = get_nhwc_from_shape(input_tensor.tensor.ShapeAsNumpy()) + _, input2_h, input2_w, input2_c = get_nhwc_from_shape(input2_tensor.tensor.ShapeAsNumpy()) + _, output_h, output_w, output_c = get_nhwc_from_shape(output_tensor.tensor.ShapeAsNumpy()) + + # Find out which tensor is the main input, we assume the 1st input is the main one + if input2_c == output_c and output_w == input2_w and output_h == input2_h: + temp = input_tensor + input2_tensor = input_tensor + input_tensor = temp + th, tw, tc = input_h, input_w, input_c + input_h, input_w, input_c = input2_h, input2_w, input2_c + input2_h, input2_w, input2_c = th, tw, tc + + assert input_h == output_h, "tensor shape not consistent" + assert input_w == output_w, "tensor shape not consistent" + assert input_c == output_c, "tensor shape not consistent" + + # tensor types + input_type = getTensorTypeStr(input_tensor.tensor.Type()) + output_type = getTensorTypeStr(output_tensor.tensor.Type()) + assert input_type == output_type, "tensor type is not consistent" + + # Check if the divisor is constant, e.g., a scale value or tensor + input2_idx = input2_tensor.tensor_idx + try: + input2 = get_np_from_wrapper(input2_tensor) + input2_idx = "constant" + except Exception: + input2 = None + + # assign params + params = { + # op related + "op": "SUB", + "input_idx": input_tensor.tensor_idx, + "input2": input2, + "input2_idx": input2_idx, + "output_idx": output_tensor.tensor_idx, + # tensor related + "input_h": input_h, + "input_w": input_w, + "input_c": input_c, + "input2_h": input2_h, + "input2_w": input2_w, + "input2_c": input2_c, + "output_h": output_h, + "output_w": output_w, + "output_c": output_c, + "input_size": input_h * input_w * input_c, + "input_dtype": input_type, + "output_dtype": output_type, + "input_meta": None, + } + op = sub.sub(params) + + return op diff --git a/code_generator/operators/__init__.py b/code_generator/operators/__init__.py index 7c167d0a..90cda31b 100644 --- a/code_generator/operators/__init__.py +++ b/code_generator/operators/__init__.py @@ -23,4 +23,11 @@ "div", "strided_slice", "se_element_mult", + "batchmatmul", + "concat", + "equal", + "softmax", + "squarddiff", + "rsqrt", + "slice", ] diff --git a/code_generator/operators/add.py b/code_generator/operators/add.py index 16a692cb..52307a31 100644 --- a/code_generator/operators/add.py +++ b/code_generator/operators/add.py @@ -1,3 +1,4 @@ +import logging import warnings from ..constant import USE_BIT_MASK @@ -16,6 +17,7 @@ "input_h": None, "input_w": None, "input_c": None, + "input2": None, "input2_dim": None, "input2_h": None, "input2_w": None, @@ -110,13 +112,16 @@ def generate_inference_str(self): + f"{self._getBufferstr(params['output2_buf_add'], params['output2_buf_add_offset'])});\n" ) else: - string += ( - f"add_fpreq({str(int(params['input_h']*params['input_w']*params['input_c']))}, " - + f"{self._getBufferstr(params['input_buf_add'], params['input_buf_add_offset'])}," - + f"{str(params['input_scale'])},{str(params['input_zero_point'])}," - + f"{self._getBufferstr(params['input2_buf_add'], params['input2_buf_add_offset'])}," - + f"{str(params['input2_scale'])},{str(params['input2_zero_point'])}," - + f"{str(params['output_scale'])},{str(params['output_zero_point'])}," - + f"{self._getBufferstr(params['output_buf_add'], params['output_buf_add_offset'])});\n" - ) + if isinstance(params["input2_idx"], str) and "constant" in params["input2_idx"]: + logging.warn("Add operator with constant support is still no ready.") + else: + string += ( + f"add_fpreq({str(int(params['input_h']*params['input_w']*params['input_c']))}, " + + f"{self._getBufferstr(params['input_buf_add'], params['input_buf_add_offset'])}," + + f"{str(params['input_scale'])},{str(params['input_zero_point'])}," + + f"{self._getBufferstr(params['input2_buf_add'], params['input2_buf_add_offset'])}," + + f"{str(params['input2_scale'])},{str(params['input2_zero_point'])}," + + f"{str(params['output_scale'])},{str(params['output_zero_point'])}," + + f"{self._getBufferstr(params['output_buf_add'], params['output_buf_add_offset'])});\n" + ) return string diff --git a/code_generator/operators/basic_utils.py b/code_generator/operators/basic_utils.py index fc81c308..f826fd23 100644 --- a/code_generator/operators/basic_utils.py +++ b/code_generator/operators/basic_utils.py @@ -265,13 +265,15 @@ def isconstanttstr(str): return False -def islabelstr(str): - if "label" in str: - return True +def islabelstr(idx_name): + if isinstance(idx_name, str): + if "label" in idx_name: + return True return False -def isParamstr(str): - if "scale" in str or "weight" in str or "bias" in str: - return True +def isParamstr(idx_name): + if isinstance(idx_name, str): + if "scale" in idx_name or "weight" in idx_name or "bias" in idx_name: + return True return False diff --git a/code_generator/operators/batchmatmul.py b/code_generator/operators/batchmatmul.py new file mode 100644 index 00000000..57d9f8ed --- /dev/null +++ b/code_generator/operators/batchmatmul.py @@ -0,0 +1,71 @@ +import logging +import warnings + +from .basic_utils import basicOperator, deep_copy_dicts, overwrite_dicts + +__all__ = ["batchmatmul"] + +default_params = { + # op related + "op": "BATCH_MATMUL", + "batch_size": None, + "input_idx": None, + "input2_idx": None, + "output_idx": None, + # tensor related + "input_col": None, + "input_row": None, + "input2": None, + "input2_col": None, + "input2_row": None, + "output_col": None, + "output_row": None, + "input_dtype": "fp32", + "input2_dtype": "fp32", + "output_dtype": "fp32", +} + + +class batchmatmul(basicOperator): + div_const_cnt = 0 + + def __init__(self, params: dict) -> None: + self.params = deep_copy_dicts(default_params) + overwrite_dicts(self.params, params) + super().__init__() + # handle input/output tensors in HWC format + self._add_input( + self.params["input_idx"], + self.params["input_dtype"], + self.params["batch_size"], + self.params["input_row"], + self.params["input_col"], + ) + self._add_input( + self.params["input2_idx"], + self.params["input2_dtype"], + self.params["batch_size"], + self.params["input2_row"], + self.params["input2_col"], + ) + self._add_output( + self.params["output_idx"], + self.params["output_dtype"], + self.params["batch_size"], + self.params["output_row"], + self.params["output_col"], + ) + + if None in default_params: + warnings.warn(f"parameters are not all set for op {self.params['op']}") + + def generate_inference_str(self): + params = self.params + string = "" + + if params["input_dtype"] == "float32": + logging.warn("BATCHMAMUL still needs implementation.") + else: + raise NotImplementedError + + return string diff --git a/code_generator/operators/cast.py b/code_generator/operators/cast.py index 52b91f62..6d667289 100644 --- a/code_generator/operators/cast.py +++ b/code_generator/operators/cast.py @@ -7,7 +7,7 @@ default_params = { # op related - "op": "ONES_LIKE", + "op": "CAST", "input_idx": None, "output_idx": None, # tensor related @@ -29,7 +29,8 @@ def __init__(self, params: dict) -> None: # raise NotImplementedError # else: if ( - "constant" not in params["input_idx"] + isinstance(params["input_idx"], str) + and "constant" not in params["input_idx"] and "weight" not in params["input_idx"] and "bias" not in params["input_idx"] ): diff --git a/code_generator/operators/concat.py b/code_generator/operators/concat.py new file mode 100644 index 00000000..a0240ef6 --- /dev/null +++ b/code_generator/operators/concat.py @@ -0,0 +1,74 @@ +import logging +import warnings + +from .basic_utils import basicOperator, deep_copy_dicts, overwrite_dicts + +__all__ = ["concat"] + +default_params = { + # op related + "op": "CONCATENATION", + "input_idx": None, + "input2_idx": None, + "output_idx": None, + # tensor related + "input_h": None, + "input_w": None, + "input_c": None, + "input2": None, + "input2_h": None, + "input2_w": None, + "input2_c": None, + "output_h": None, + "output_w": None, + "output_c": None, + "input_dtype": None, + "input2_dtype": None, + "output_dtype": None, +} + + +class concat(basicOperator): + div_const_cnt = 0 + + def __init__(self, params: dict) -> None: + self.params = deep_copy_dicts(default_params) + overwrite_dicts(self.params, params) + super().__init__() + # handle input/output tensors in HWC format + self._add_input( + self.params["input_idx"], + self.params["input_dtype"], + self.params["input_h"], + self.params["input_w"], + self.params["input_c"], + ) + if "constant" not in self.params["input2_idx"]: + self._add_input( + self.params["input2_idx"], + self.params["input2_dtype"], + self.params["input2_h"], + self.params["input2_w"], + self.params["input2_c"], + ) + self._add_output( + self.params["output_idx"], + self.params["output_dtype"], + self.params["output_h"], + self.params["output_w"], + self.params["output_c"], + ) + + if None in default_params: + warnings.warn(f"parameters are not all set for op {self.params['op']}") + + def generate_inference_str(self): + params = self.params + string = "" + + if params["input_dtype"] == "float32": + logging.warn("CONCATENATION still needs implementation.") + else: + raise NotImplementedError + + return string diff --git a/code_generator/operators/div.py b/code_generator/operators/div.py index 227c0bfd..0378bd7c 100644 --- a/code_generator/operators/div.py +++ b/code_generator/operators/div.py @@ -14,6 +14,15 @@ "output_idx": None, # tensor related "input_size": None, + "input_h": None, + "input_w": None, + "input_c": None, + "input2_h": None, + "input2_w": None, + "input2_c": None, + "output_h": None, + "output_w": None, + "output_c": None, "input2": None, "input_dtype": "int8", "input2_dtype": "int8", @@ -41,7 +50,8 @@ def __init__(self, params: dict) -> None: super().__init__() # handle input/output tensors in HWC format self._add_input(self.params["input_idx"], self.params["input_dtype"], self.params["input_size"], 1, 1) - self._add_input(self.params["input2_idx"], self.params["input2_dtype"], self.params["input_size"], 1, 1) + if "constant" not in self.params["input2_idx"]: + self._add_input(self.params["input2_idx"], self.params["input2_dtype"], self.params["input_size"], 1, 1) self._add_output(self.params["output_idx"], self.params["output_dtype"], self.params["input_size"], 1, 1) if None in default_params: diff --git a/code_generator/operators/equal.py b/code_generator/operators/equal.py new file mode 100644 index 00000000..a267b2fc --- /dev/null +++ b/code_generator/operators/equal.py @@ -0,0 +1,71 @@ +import logging +import warnings + +from .basic_utils import basicOperator, deep_copy_dicts, overwrite_dicts + +__all__ = ["equal"] + +default_params = { + # op related + "op": "EQUAL", + "input_idx": None, + "input2_idx": None, + "output_idx": None, + # tensor related + "input_h": None, + "input_w": None, + "input_c": None, + "input2": None, + "input2_h": None, + "input2_w": None, + "input2_c": None, + "output_h": None, + "output_w": None, + "output_c": None, + "input_dtype": None, + "input2_dtype": None, + "output_dtype": None, +} + + +class equal(basicOperator): + def __init__(self, params: dict) -> None: + self.params = deep_copy_dicts(default_params) + overwrite_dicts(self.params, params) + super().__init__() + # handle input/output tensors in HWC format + self._add_input( + self.params["input_idx"], + self.params["input_dtype"], + self.params["input_h"], + self.params["input_w"], + self.params["input_c"], + ) + self._add_input( + self.params["input2_idx"], + self.params["input2_dtype"], + self.params["input2_h"], + self.params["input2_w"], + self.params["input2_c"], + ) + self._add_output( + self.params["output_idx"], + self.params["output_dtype"], + self.params["output_h"], + self.params["output_w"], + self.params["output_c"], + ) + + if None in default_params: + warnings.warn(f"parameters are not all set for op {self.params['op']}") + + def generate_inference_str(self): + params = self.params + string = "" + + if params["input_dtype"] == "float32": + logging.warn("EQUAL still needs implementation.") + else: + raise NotImplementedError + + return string diff --git a/code_generator/operators/mul.py b/code_generator/operators/mul.py index ce272812..828c08db 100644 --- a/code_generator/operators/mul.py +++ b/code_generator/operators/mul.py @@ -43,7 +43,7 @@ def __init__(self, params: dict) -> None: # handle input/output tensors in HWC format self._add_input(self.params["input_idx"], self.params["input_dtype"], self.params["input_size"], 1, 1) if not (isParamstr(self.params["input2_idx"]) or islabelstr(self.params["input2_idx"])): - self._add_input(self.params["input2_idx"], self.params["input2_dtype"], self.params["output_size"], 1, 1) + self._add_input(self.params["input2_idx"], self.params["input2_dtype"], self.params["input2_size"], 1, 1) self._add_output(self.params["output_idx"], self.params["output_dtype"], self.params["output_size"], 1, 1) if None in default_params: diff --git a/code_generator/operators/notequal.py b/code_generator/operators/notequal.py new file mode 100644 index 00000000..d45aed44 --- /dev/null +++ b/code_generator/operators/notequal.py @@ -0,0 +1,71 @@ +import logging +import warnings + +from .basic_utils import basicOperator, deep_copy_dicts, overwrite_dicts + +__all__ = ["notequal"] + +default_params = { + # op related + "op": "NOT_EQUAL", + "input_idx": None, + "input2_idx": None, + "output_idx": None, + # tensor related + "input_h": None, + "input_w": None, + "input_c": None, + "input2": None, + "input2_h": None, + "input2_w": None, + "input2_c": None, + "output_h": None, + "output_w": None, + "output_c": None, + "input_dtype": None, + "input2_dtype": None, + "output_dtype": None, +} + + +class notequal(basicOperator): + def __init__(self, params: dict) -> None: + self.params = deep_copy_dicts(default_params) + overwrite_dicts(self.params, params) + super().__init__() + # handle input/output tensors in HWC format + self._add_input( + self.params["input_idx"], + self.params["input_dtype"], + self.params["input_h"], + self.params["input_w"], + self.params["input_c"], + ) + self._add_input( + self.params["input2_idx"], + self.params["input2_dtype"], + self.params["input2_h"], + self.params["input2_w"], + self.params["input2_c"], + ) + self._add_output( + self.params["output_idx"], + self.params["output_dtype"], + self.params["output_h"], + self.params["output_w"], + self.params["output_c"], + ) + + if None in default_params: + warnings.warn(f"parameters are not all set for op {self.params['op']}") + + def generate_inference_str(self): + params = self.params + string = "" + + if params["input_dtype"] == "float32": + logging.warn("NOT_EQUAL still needs implementation.") + else: + raise NotImplementedError + + return string diff --git a/code_generator/operators/rsqrt.py b/code_generator/operators/rsqrt.py new file mode 100644 index 00000000..16717902 --- /dev/null +++ b/code_generator/operators/rsqrt.py @@ -0,0 +1,52 @@ +import logging +import warnings + +from .basic_utils import basicOperator, deep_copy_dicts, overwrite_dicts + +__all__ = ["rsqrt"] + +default_params = { + # op related + "op": "RSQRT", + "input_idx": None, + "output_idx": None, + # tensor related + "input_dim": None, + "input_size": None, + "output_dim": None, + "input_dtype": "float32", + "output_dtype": "float32", + "input_meta": None, +} + + +class rsqrt(basicOperator): + def __init__(self, params: dict) -> None: + self.params = deep_copy_dicts(default_params) + overwrite_dicts(self.params, params) + super().__init__() + self._add_input( + self.params["input_idx"], + self.params["input_dtype"], + self.params["input_size"], + 1, + 1, + ) + self._add_output( + self.params["output_idx"], + self.params["output_dtype"], + self.params["input_size"], + 1, + 1, + ) + + if None in default_params: + warnings.warn(f"parameters are not all set for op {self.params['op']}") + + def generate_inference_str(self): + # params = self.params + string = "" + + logging.warn("RSQRT operator support is still no ready.") + + return string diff --git a/code_generator/operators/slice.py b/code_generator/operators/slice.py new file mode 100644 index 00000000..5aabb043 --- /dev/null +++ b/code_generator/operators/slice.py @@ -0,0 +1,59 @@ +import logging + +from .basic_utils import basicOperator, deep_copy_dicts, overwrite_dicts + +__all__ = ["slice"] + +default_params = { + # op related + "op": "SLICE", + "input_idx": None, + "output_idx": None, + # tensor related + "input_h": None, + "input_w": None, + "input_c": None, + "begin_h": None, + "begin_w": None, + "begin_c": None, + "end_h": None, + "end_w": None, + "end_c": None, + "output_h": None, + "output_w": None, + "output_c": None, + "input_dtype": "fp32", + "output_dtype": "fp32", +} + + +class slice(basicOperator): + div_const_cnt = 0 + + def __init__(self, params: dict) -> None: + self.params = deep_copy_dicts(default_params) + overwrite_dicts(self.params, params) + super().__init__() + # handle input/output tensors in HWC format + self._add_input( + self.params["input_idx"], + self.params["input_dtype"], + self.params["input_h"], + self.params["input_w"], + self.params["input_c"], + ) + self._add_output( + self.params["output_idx"], + self.params["output_dtype"], + self.params["output_h"], + self.params["output_w"], + self.params["output_c"], + ) + + def generate_inference_str(self): + # params = self.params + string = "" + + logging.warn("slice operator is still no ready.") + + return string diff --git a/code_generator/operators/softmax.py b/code_generator/operators/softmax.py new file mode 100644 index 00000000..177584c1 --- /dev/null +++ b/code_generator/operators/softmax.py @@ -0,0 +1,81 @@ +# ---------------------------------------------------------------------- +# Project: TinyEngine +# Title: softmax.py +# +# Reference papers: +# - MCUNet: Tiny Deep Learning on IoT Device, NeurIPS 2020 +# - MCUNetV2: Memory-Efficient Patch-based Inference for Tiny Deep Learning, NeurIPS 2021 +# - MCUNetV3: On-Device Training Under 256KB Memory, NeurIPS 2022 +# Contact authors: +# - Wei-Ming Chen, wmchen@mit.edu +# - Wei-Chen Wang, wweichen@mit.edu +# - Ji Lin, jilin@mit.edu +# - Ligeng Zhu, ligeng@mit.edu +# - Song Han, songhan@mit.edu +# +# Target ISA: ARMv7E-M +# ---------------------------------------------------------------------- + +import logging +import warnings + +from .basic_utils import basicOperator, deep_copy_dicts, overwrite_dicts + +__all__ = ["softmax"] + + +default_params = { + # op related + "op": "SOFTMAX", + "input_idx": None, + "output_idx": None, + # tensor related + "input_dim": None, + "input_h": None, + "input_w": None, + "input_c": None, + "output_dim": None, + "output_h": None, + "output_w": None, + "output_c": None, + "input_dtype": "int8", + "output_dtype": "int8", + # quantization related + "input_zero_point": None, + "output_zero_point": None, + "input_scale": None, + "output_scale": None, +} + + +class softmax(basicOperator): + def __init__(self, params: dict) -> None: + self.params = deep_copy_dicts(default_params) + overwrite_dicts(self.params, params) + super().__init__() + # handle input/output tensors in HWC format + self._add_input( + self.params["input_idx"], + self.params["input_dtype"], + self.params["input_h"], + self.params["input_w"], + self.params["input_c"], + ) + self._add_output( + self.params["output_idx"], + self.params["output_dtype"], + self.params["output_h"], + self.params["output_w"], + self.params["output_c"], + ) + + if None in default_params: + warnings.warn(f"parameters are not all set for op {self.params['op']}") + + def generate_inference_str(self): + # params = self.params + string = "" + + logging.warn("Add operator with constant support is still no ready.") + + return string diff --git a/code_generator/operators/squarddiff.py b/code_generator/operators/squarddiff.py new file mode 100644 index 00000000..cdf7ff68 --- /dev/null +++ b/code_generator/operators/squarddiff.py @@ -0,0 +1,75 @@ +import logging +import warnings + +from .basic_utils import basicOperator, deep_copy_dicts, overwrite_dicts + +__all__ = ["squarddiff"] + +default_params = { + # op related + "op": "SQUARED_DIFFERENCE", + "input_idx": None, + "input2_idx": None, + "output_idx": None, + # tensor related + "input_size": None, + "input_h": None, + "input_w": None, + "input_c": None, + "input2_h": None, + "input2_w": None, + "input2_c": None, + "output_h": None, + "output_w": None, + "output_c": None, + "input2": None, + "input_dtype": "fp32", + "input2_dtype": "fp32", + "output_dtype": "fp32", +} + + +class squarddiff(basicOperator): + div_const_cnt = 0 + + def __init__(self, params: dict) -> None: + self.params = deep_copy_dicts(default_params) + overwrite_dicts(self.params, params) + super().__init__() + # handle input/output tensors in HWC format + self._add_input( + self.params["input_idx"], + self.params["input_dtype"], + self.params["input_h"], + self.params["input_w"], + self.params["input_c"], + ) + if isinstance(self.params["input2_idx"], str) and "constant" not in self.params["input2_idx"]: + self._add_input( + self.params["input2_idx"], + self.params["input2_dtype"], + self.params["input2_h"], + self.params["input2_w"], + self.params["input2_c"], + ) + self._add_output( + self.params["output_idx"], + self.params["output_dtype"], + self.params["output_h"], + self.params["output_w"], + self.params["output_c"], + ) + + if None in default_params: + warnings.warn(f"parameters are not all set for op {self.params['op']}") + + def generate_inference_str(self): + params = self.params + string = "" + + if params["input_dtype"] == "float32": + logging.warn("squarddiff operator with constant support is still no ready.") + else: + raise NotImplementedError + + return string diff --git a/code_generator/operators/sub.py b/code_generator/operators/sub.py index 28c3bd9a..891e56ad 100644 --- a/code_generator/operators/sub.py +++ b/code_generator/operators/sub.py @@ -1,3 +1,4 @@ +import logging import warnings from .basic_utils import basicOperator, deep_copy_dicts, overwrite_dicts @@ -8,6 +9,7 @@ # op related "op": "SUB", "input_idx": None, + "input2": None, "input2_idx": None, "output_idx": None, # tensor related @@ -42,6 +44,10 @@ def __init__(self, params: dict) -> None: def generate_inference_str(self): params = self.params + + if isinstance(params["input2_idx"], str) and "constant" in params["input2_idx"]: + logging.warn("Please implement sub operator with a constant tensor.") + if params["input_dtype"] == "float32": string = ( f"sub({self.params['input_size']}," From 199177c6de3d4ba1c7c281c1c7b29319786dba69 Mon Sep 17 00:00:00 2001 From: meenchen Date: Thu, 23 Feb 2023 16:59:17 -0500 Subject: [PATCH 03/19] test --- code_generator/operators/mul.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code_generator/operators/mul.py b/code_generator/operators/mul.py index 828c08db..ce272812 100644 --- a/code_generator/operators/mul.py +++ b/code_generator/operators/mul.py @@ -43,7 +43,7 @@ def __init__(self, params: dict) -> None: # handle input/output tensors in HWC format self._add_input(self.params["input_idx"], self.params["input_dtype"], self.params["input_size"], 1, 1) if not (isParamstr(self.params["input2_idx"]) or islabelstr(self.params["input2_idx"])): - self._add_input(self.params["input2_idx"], self.params["input2_dtype"], self.params["input2_size"], 1, 1) + self._add_input(self.params["input2_idx"], self.params["input2_dtype"], self.params["output_size"], 1, 1) self._add_output(self.params["output_idx"], self.params["output_dtype"], self.params["output_size"], 1, 1) if None in default_params: From bd17ba622222c1b8c0f6ebc1c3677cc11d778ebd Mon Sep 17 00:00:00 2001 From: meenchen Date: Thu, 23 Feb 2023 17:09:56 -0500 Subject: [PATCH 04/19] fix skip --- code_generator/TfliteConvertor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code_generator/TfliteConvertor.py b/code_generator/TfliteConvertor.py index b9084730..db38ab75 100644 --- a/code_generator/TfliteConvertor.py +++ b/code_generator/TfliteConvertor.py @@ -171,7 +171,7 @@ def _handleOperator(self, op): self.layer.append(TF_Parser.parse_rsqrt(op, self.model)) elif op_code_str == "SLICE": self.layer.append(TF_Parser.parse_slice(op, self.model)) - elif op_code_str == SKIP_OPs: + elif op_code_str in SKIP_OPs: pass else: raise NotImplementedError(f"Unsupported {op_code_str}") From 2233ddc0520a4f33e4cee089c3da76d4d9df21fa Mon Sep 17 00:00:00 2001 From: meenchen Date: Thu, 23 Feb 2023 17:18:11 -0500 Subject: [PATCH 05/19] fix add --- code_generator/converters/tflite_parser/add.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code_generator/converters/tflite_parser/add.py b/code_generator/converters/tflite_parser/add.py index 33ea5f3e..92803777 100644 --- a/code_generator/converters/tflite_parser/add.py +++ b/code_generator/converters/tflite_parser/add.py @@ -37,7 +37,7 @@ def parse_add(op, model: Model.Model): _, output_h, output_w, output_c = get_nhwc_from_shape(output_tensor.tensor.ShapeAsNumpy()) # Find out which tensor is the main input, we assume the 1st input is the main one - if input2_c == output_c and output_w == input2_w and output_h == input2_h: + if input_c != output_c or output_w != input_w or output_h != input2_h: temp = input_tensor input2_tensor = input_tensor input_tensor = temp From afc93e307123e127de146b219fe44fe4d0503484 Mon Sep 17 00:00:00 2001 From: meenchen Date: Thu, 23 Feb 2023 17:29:31 -0500 Subject: [PATCH 06/19] typo --- code_generator/converters/tflite_parser/add.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code_generator/converters/tflite_parser/add.py b/code_generator/converters/tflite_parser/add.py index 92803777..197a8c5f 100644 --- a/code_generator/converters/tflite_parser/add.py +++ b/code_generator/converters/tflite_parser/add.py @@ -37,7 +37,7 @@ def parse_add(op, model: Model.Model): _, output_h, output_w, output_c = get_nhwc_from_shape(output_tensor.tensor.ShapeAsNumpy()) # Find out which tensor is the main input, we assume the 1st input is the main one - if input_c != output_c or output_w != input_w or output_h != input2_h: + if input_c != output_c or output_w != input_w or output_h != input_h: temp = input_tensor input2_tensor = input_tensor input_tensor = temp From 38894200ac19b70b0efde819c66c5b5259d1cefe Mon Sep 17 00:00:00 2001 From: meenchen Date: Thu, 23 Feb 2023 18:40:25 -0500 Subject: [PATCH 07/19] skeleton --- code_generator/converters/tflite_parser/fc.py | 2 ++ code_generator/converters/tflite_parser/notequal.py | 2 +- code_generator/converters/tflite_parser/utils.py | 5 +++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/code_generator/converters/tflite_parser/fc.py b/code_generator/converters/tflite_parser/fc.py index cc16d89c..4fcaa469 100644 --- a/code_generator/converters/tflite_parser/fc.py +++ b/code_generator/converters/tflite_parser/fc.py @@ -89,6 +89,8 @@ def parse_fc(op, model: Model.Model): "dtypte": input_type, "kernel_h": 1, "kernel_w": 1, + "stride_h": 1, + "stride_w": 1, # trainable parameters "weight_value": weight, "bias": bias, diff --git a/code_generator/converters/tflite_parser/notequal.py b/code_generator/converters/tflite_parser/notequal.py index c44c4985..505eb50a 100644 --- a/code_generator/converters/tflite_parser/notequal.py +++ b/code_generator/converters/tflite_parser/notequal.py @@ -15,7 +15,7 @@ def parse_notequal(op, model: Model.Model): input_type = getTensorTypeStr(input_tensors[0].tensor.Type()) input2_type = getTensorTypeStr(input_tensors[1].tensor.Type()) output_type = getTensorTypeStr(output_tensor.tensor.Type()) - assert input_type == output_type == input2_type, "tensor type not consistent" + assert input_type == input2_type, "tensor type not consistent" # shapes _, input_h, input_w, input_c = get_nhwc_from_shape(input_tensors[0].tensor.ShapeAsNumpy()) diff --git a/code_generator/converters/tflite_parser/utils.py b/code_generator/converters/tflite_parser/utils.py index fd9afb7e..3010218b 100644 --- a/code_generator/converters/tflite_parser/utils.py +++ b/code_generator/converters/tflite_parser/utils.py @@ -144,6 +144,11 @@ def getTensorTypeStr(type): return "uint8" if TensorType.FLOAT32 == type: return "float32" + if TensorType.INT32 == type: + return "int32" + if TensorType.BOOL == type: + return "bool" + raise NotImplementedError def get_hwc_from_chwshape(shape): From fa054511bc1281e852c7a1f30e44cc9a20af9af6 Mon Sep 17 00:00:00 2001 From: meenchen Date: Fri, 24 Feb 2023 11:26:55 -0500 Subject: [PATCH 08/19] fix bug --- code_generator/CodeGenerator.py | 13 +++++++------ .../converters/tflite_parser/batchmatmul.py | 6 +++--- code_generator/converters/tflite_parser/fc.py | 3 ++- code_generator/operators/basic_utils.py | 8 ++++++++ code_generator/operators/cast.py | 15 +++++++-------- code_generator/operators/equal.py | 2 +- code_generator/operators/notequal.py | 2 +- 7 files changed, 29 insertions(+), 20 deletions(-) diff --git a/code_generator/CodeGenerator.py b/code_generator/CodeGenerator.py index ac5f7f0b..d10a484f 100644 --- a/code_generator/CodeGenerator.py +++ b/code_generator/CodeGenerator.py @@ -585,12 +585,13 @@ def _parseTrainable(self): ) else: self._parseBias(self.parse_count, layer_info["bias"].flatten()) - self._parseEffectivescales(self.parse_count, layer_info["effective_scale"].flatten()) - self._parseRequantize( - self.parse_count, - layer_info["shift"].flatten(), - layer_info["multiplier"].flatten(), - ) + if layer_info["input_dtype"] == "int8": + self._parseEffectivescales(self.parse_count, layer_info["effective_scale"].flatten()) + self._parseRequantize( + self.parse_count, + layer_info["shift"].flatten(), + layer_info["multiplier"].flatten(), + ) layer_info["parsed_trainable"] = self.parse_count self.parse_count += 1 diff --git a/code_generator/converters/tflite_parser/batchmatmul.py b/code_generator/converters/tflite_parser/batchmatmul.py index 152ed312..0aac4619 100644 --- a/code_generator/converters/tflite_parser/batchmatmul.py +++ b/code_generator/converters/tflite_parser/batchmatmul.py @@ -47,9 +47,9 @@ def parse_batchmatmul(op, model: Model.Model): "input2_row": input2_row, "output_col": output_col, "output_row": output_row, - "input_dtype": "fp32", - "input2_dtype": "fp32", - "output_dtype": "fp32", + "input_dtype": input_type, + "input2_dtype": input2_type, + "output_dtype": output_type, } op = batchmatmul.batchmatmul(params) diff --git a/code_generator/converters/tflite_parser/fc.py b/code_generator/converters/tflite_parser/fc.py index 4fcaa469..3aa9281b 100644 --- a/code_generator/converters/tflite_parser/fc.py +++ b/code_generator/converters/tflite_parser/fc.py @@ -86,7 +86,8 @@ def parse_fc(op, model: Model.Model): "output_h": output_h, "output_w": 1, "output_c": output_c, - "dtypte": input_type, + "input_dtype": input_type, + "output_dtype": output_type, "kernel_h": 1, "kernel_w": 1, "stride_h": 1, diff --git a/code_generator/operators/basic_utils.py b/code_generator/operators/basic_utils.py index f826fd23..61611c7b 100644 --- a/code_generator/operators/basic_utils.py +++ b/code_generator/operators/basic_utils.py @@ -183,6 +183,14 @@ def _getBufferstrCast(self, location, offset, dtype="float32"): class tensor: + size: int + dtype: str + buffer_placement: int + buffer_name: str + buffer_address: int + allocator_idx: str + graph_idx: str + byte_size = { "bool": 1, "int8": 1, diff --git a/code_generator/operators/cast.py b/code_generator/operators/cast.py index 6d667289..9937758b 100644 --- a/code_generator/operators/cast.py +++ b/code_generator/operators/cast.py @@ -28,12 +28,8 @@ def __init__(self, params: dict) -> None: # handle input/output tensors in HWC format # raise NotImplementedError # else: - if ( - isinstance(params["input_idx"], str) - and "constant" not in params["input_idx"] - and "weight" not in params["input_idx"] - and "bias" not in params["input_idx"] - ): + input_idx_str = str(params["input_idx"]) + if "constant" not in input_idx_str and "weight" not in input_idx_str and "bias" not in input_idx_str: self._add_input( self.params["input_idx"], self.params["input_dtype"], @@ -62,6 +58,9 @@ def generate_inference_str(self): elif params["input_dtype"] == "int32" or params["input_dtype"] == "int8": input_type_str = "int8_t" input_ptr = "int8ptr" + elif params["input_dtype"] == "bool": + input_type_str = "int8_t" + input_ptr = "int8ptr" else: raise NotImplementedError if params["output_dtype"] == "float32": @@ -73,7 +72,7 @@ def generate_inference_str(self): else: raise NotImplementedError # generate string - if "constant" in params["input_idx"] and "data" in params["input_meta"]: + if "constant" in str(params["input_idx"]) and "data" in params["input_meta"]: array_name = params["input_idx"].replace("@", "").replace("-", "") array = "{" array_len = 0 @@ -91,7 +90,7 @@ def generate_inference_str(self): + f"{self._getBufferstr(params['output_buf_add'], params['output_buf_add_offset'])};\n" + f"for(int i = 0; i < {array_len}; i++) {output_ptr}[i] = ({output_type_str}){array_name}[i];\n" ) - elif "weight" in params["input_idx"]: + elif "weight" in str(params["input_idx"]): string = ( f"{output_ptr} = ({output_type_str}*)" + f"{self._getBufferstr(params['output_buf_add'], params['output_buf_add_offset'])};\n" diff --git a/code_generator/operators/equal.py b/code_generator/operators/equal.py index a267b2fc..1aa25e78 100644 --- a/code_generator/operators/equal.py +++ b/code_generator/operators/equal.py @@ -63,7 +63,7 @@ def generate_inference_str(self): params = self.params string = "" - if params["input_dtype"] == "float32": + if params["input_dtype"] in ["float32", "int32", "bool"]: logging.warn("EQUAL still needs implementation.") else: raise NotImplementedError diff --git a/code_generator/operators/notequal.py b/code_generator/operators/notequal.py index d45aed44..682c3df6 100644 --- a/code_generator/operators/notequal.py +++ b/code_generator/operators/notequal.py @@ -63,7 +63,7 @@ def generate_inference_str(self): params = self.params string = "" - if params["input_dtype"] == "float32": + if params["input_dtype"] in ["float32", "int32", "bool"]: logging.warn("NOT_EQUAL still needs implementation.") else: raise NotImplementedError From d7fe812c033a7c6a0d679bb10fdfa2b47234204f Mon Sep 17 00:00:00 2001 From: meenchen Date: Fri, 24 Feb 2023 14:03:44 -0500 Subject: [PATCH 09/19] parse constant --- code_generator/CodeGenerator.py | 33 +++++++++++++++++-- .../converters/tflite_parser/add.py | 2 +- .../converters/tflite_parser/concat.py | 2 +- .../converters/tflite_parser/div.py | 2 +- .../converters/tflite_parser/equal.py | 2 +- .../converters/tflite_parser/mul.py | 2 +- .../converters/tflite_parser/notequal.py | 2 +- .../converters/tflite_parser/squarddiff.py | 2 +- .../converters/tflite_parser/sub.py | 3 +- code_generator/operators/add.py | 3 ++ code_generator/operators/basic_utils.py | 17 ++++++++++ code_generator/operators/mul.py | 6 +++- code_generator/operators/notequal.py | 3 ++ code_generator/operators/sub.py | 5 ++- 14 files changed, 72 insertions(+), 12 deletions(-) diff --git a/code_generator/CodeGenerator.py b/code_generator/CodeGenerator.py index d10a484f..9fd1b687 100644 --- a/code_generator/CodeGenerator.py +++ b/code_generator/CodeGenerator.py @@ -18,7 +18,10 @@ import os +import numpy as np + from .constant import FUSE_SGD_UPDATE_STR, FUSHION_CONFIG +from .operators.basic_utils import tensor from .OpGenerator import OpGenerator Codegen_root = "./codegen/" @@ -34,8 +37,6 @@ class CodeGenerator: """Provide utilities to generate C code for a given model and memory schdeule.""" parse_count = 0 - header_handle = None - source_handle = None def __init__( self, @@ -771,6 +772,34 @@ def _parseTrainable(self): layer_info["parsed_trainable"] = self.parse_count self.parse_count += 1 + else: + # Parse constants of inputs + for t in op.input_tensors: + if t.constant(): + if t.data is None: + raise ValueError("constant tensor data not found") + self._parseConstant(tensor) + + def _parseConstant(self, t: tensor): + def type_to_c_type(type: str) -> str: + if type == "int8": + return "unsigned char" + elif type == "float32": + return "float" + elif type == "int32": + return "int32_t" + elif type == "bool": + return "char" # Using bytes to store boolean + raise NotImplementedError + + fp = self.header_handle + # 8bit implementation + string = f"const {type_to_c_type(t.dtype)}" + "[" + str(np.prod(t.size)) + "] = {" + flat_data = t.data.flatten() + for d in flat_data: + string += f"{d}, " + string += "};\n" + fp.write(string) def _parseCWHWeight(self, Lindex, weight, height, width, channel): fp = self.header_handle diff --git a/code_generator/converters/tflite_parser/add.py b/code_generator/converters/tflite_parser/add.py index 197a8c5f..9a491e23 100644 --- a/code_generator/converters/tflite_parser/add.py +++ b/code_generator/converters/tflite_parser/add.py @@ -59,7 +59,7 @@ def parse_add(op, model: Model.Model): input2_idx = input2_tensor.tensor_idx try: input2 = get_np_from_wrapper(input2_tensor) - input2_idx = "constant" + input2_idx = "constant" + input2_idx except Exception: input2 = None diff --git a/code_generator/converters/tflite_parser/concat.py b/code_generator/converters/tflite_parser/concat.py index 726b3ad7..af7a75fa 100644 --- a/code_generator/converters/tflite_parser/concat.py +++ b/code_generator/converters/tflite_parser/concat.py @@ -46,7 +46,7 @@ def parse_concat(op, model: Model.Model): input2_idx = input2_tensor.tensor_idx try: input2 = get_np_from_wrapper(input2_tensor) - input2_idx = "constant" + input2_idx = "constant" + input2_idx except Exception: input2 = None diff --git a/code_generator/converters/tflite_parser/div.py b/code_generator/converters/tflite_parser/div.py index 795452c2..d9cf2a97 100644 --- a/code_generator/converters/tflite_parser/div.py +++ b/code_generator/converters/tflite_parser/div.py @@ -46,7 +46,7 @@ def parse_div(op, model: Model.Model): input2_idx = input2_tensor.tensor_idx try: input2 = get_np_from_wrapper(input2_tensor) - input2_idx = "constant" + input2_idx = "constant" + str(input2_idx) except Exception: input2 = None diff --git a/code_generator/converters/tflite_parser/equal.py b/code_generator/converters/tflite_parser/equal.py index 8fb1493d..9aabb53a 100644 --- a/code_generator/converters/tflite_parser/equal.py +++ b/code_generator/converters/tflite_parser/equal.py @@ -26,7 +26,7 @@ def parse_equal(op, model: Model.Model): input2_idx = input_tensors[1].tensor_idx try: input2 = get_np_from_wrapper(input_tensors[1]) - input2_idx = "constant" + input2_idx = "constant" + str(input2_idx) except Exception: input2 = None diff --git a/code_generator/converters/tflite_parser/mul.py b/code_generator/converters/tflite_parser/mul.py index 18282761..27d81265 100644 --- a/code_generator/converters/tflite_parser/mul.py +++ b/code_generator/converters/tflite_parser/mul.py @@ -40,7 +40,7 @@ def parse_mul(op, model: Model.Model): input2_idx = input2_tensor.tensor_idx try: input2 = get_np_from_wrapper(input2_tensor) - input2_idx = "constant" + input2_idx = "constant" + str(input2_idx) except Exception: input2 = None diff --git a/code_generator/converters/tflite_parser/notequal.py b/code_generator/converters/tflite_parser/notequal.py index 505eb50a..1469c312 100644 --- a/code_generator/converters/tflite_parser/notequal.py +++ b/code_generator/converters/tflite_parser/notequal.py @@ -26,7 +26,7 @@ def parse_notequal(op, model: Model.Model): input2_idx = input_tensors[1].tensor_idx try: input2 = get_np_from_wrapper(input_tensors[1]) - input2_idx = "constant" + input2_idx = "constant" + str(input2_idx) except Exception: input2 = None diff --git a/code_generator/converters/tflite_parser/squarddiff.py b/code_generator/converters/tflite_parser/squarddiff.py index dbf87976..0c903790 100644 --- a/code_generator/converters/tflite_parser/squarddiff.py +++ b/code_generator/converters/tflite_parser/squarddiff.py @@ -40,7 +40,7 @@ def parse_squarddiff(op, model: Model.Model): input2_idx = input2_tensor.tensor_idx try: input2 = get_np_from_wrapper(input2_tensor) - input2_idx = "constant" + input2_idx = "constant" + str(input2_idx) except Exception: input2 = None diff --git a/code_generator/converters/tflite_parser/sub.py b/code_generator/converters/tflite_parser/sub.py index a109f99d..821b31db 100644 --- a/code_generator/converters/tflite_parser/sub.py +++ b/code_generator/converters/tflite_parser/sub.py @@ -40,7 +40,7 @@ def parse_sub(op, model: Model.Model): input2_idx = input2_tensor.tensor_idx try: input2 = get_np_from_wrapper(input2_tensor) - input2_idx = "constant" + input2_idx = "constant" + str(input2_idx) except Exception: input2 = None @@ -63,6 +63,7 @@ def parse_sub(op, model: Model.Model): "output_w": output_w, "output_c": output_c, "input_size": input_h * input_w * input_c, + "input2_size": input2_h * input2_w * input2_c, "input_dtype": input_type, "output_dtype": output_type, "input_meta": None, diff --git a/code_generator/operators/add.py b/code_generator/operators/add.py index 52307a31..ff2c913c 100644 --- a/code_generator/operators/add.py +++ b/code_generator/operators/add.py @@ -76,6 +76,9 @@ def __init__(self, params: dict) -> None: self.params["input2_w"], self.params["input2_h"], ) + # TODO: Refactor this + if self.input_tensors[1].constant(): + self.input_tensors[1].set_data(self.params["input2"]) self._add_output( self.params["output_idx"], self.params["output_dtype"], diff --git a/code_generator/operators/basic_utils.py b/code_generator/operators/basic_utils.py index 61611c7b..ade40470 100644 --- a/code_generator/operators/basic_utils.py +++ b/code_generator/operators/basic_utils.py @@ -18,6 +18,7 @@ import math from copy import deepcopy +from typing import List import numpy as np @@ -44,6 +45,9 @@ class basicOperator: 4. Generate the corresponding kernel code (optional) """ + input_tensors: List + output_tensors: List + def __init__(self) -> None: self.input_tensors = [] self.output_tensors = [] @@ -190,6 +194,8 @@ class tensor: buffer_address: int allocator_idx: str graph_idx: str + is_constant: bool + data: np.ndarray = None byte_size = { "bool": 1, @@ -218,6 +224,17 @@ def __init__(self, graph_idx, dtype, dims) -> None: self.buffer_address = None self.allocator_idx = None self.graph_idx = str(graph_idx) + # TODO: This constant logic is only temporary solution, need to refactor this part + if "weight" in str(graph_idx) or "constant" in str(graph_idx): + self.is_constant = True + else: + self.is_constant = False + + def set_data(self, data: np.ndarray): + self.data = data + + def constant(self): + return self.is_constant def input_c(self): return self.size[0] diff --git a/code_generator/operators/mul.py b/code_generator/operators/mul.py index ce272812..0a3fad49 100644 --- a/code_generator/operators/mul.py +++ b/code_generator/operators/mul.py @@ -11,6 +11,7 @@ "input2_idx": None, "output_idx": None, # tensor related + "input2": None, "input_size": None, "input2_size": None, "output_size": None, @@ -43,7 +44,10 @@ def __init__(self, params: dict) -> None: # handle input/output tensors in HWC format self._add_input(self.params["input_idx"], self.params["input_dtype"], self.params["input_size"], 1, 1) if not (isParamstr(self.params["input2_idx"]) or islabelstr(self.params["input2_idx"])): - self._add_input(self.params["input2_idx"], self.params["input2_dtype"], self.params["output_size"], 1, 1) + self._add_input(self.params["input2_idx"], self.params["input2_dtype"], self.params["input2_size"], 1, 1) + # TODO: Refactor this + if self.input_tensors[1].constant(): + self.input_tensors[1].set_data(self.params["input2"]) self._add_output(self.params["output_idx"], self.params["output_dtype"], self.params["output_size"], 1, 1) if None in default_params: diff --git a/code_generator/operators/notequal.py b/code_generator/operators/notequal.py index 682c3df6..3129c1fc 100644 --- a/code_generator/operators/notequal.py +++ b/code_generator/operators/notequal.py @@ -48,6 +48,9 @@ def __init__(self, params: dict) -> None: self.params["input2_w"], self.params["input2_c"], ) + # TODO: Refactor this + if self.input_tensors[1].constant(): + self.input_tensors[1].set_data(self.params["input2"]) self._add_output( self.params["output_idx"], self.params["output_dtype"], diff --git a/code_generator/operators/sub.py b/code_generator/operators/sub.py index 891e56ad..d1ebfd6e 100644 --- a/code_generator/operators/sub.py +++ b/code_generator/operators/sub.py @@ -14,6 +14,7 @@ "output_idx": None, # tensor related "input_size": None, + "input2_size": None, "input_dtype": "float32", "input2_dtype": "float32", "output_dtype": "float32", @@ -36,7 +37,9 @@ def __init__(self, params: dict) -> None: super().__init__() # handle input/output tensors in HWC format self._add_input(self.params["input_idx"], self.params["input_dtype"], self.params["input_size"], 1, 1) - self._add_input(self.params["input2_idx"], self.params["input2_dtype"], self.params["input_size"], 1, 1) + self._add_input(self.params["input2_idx"], self.params["input2_dtype"], self.params["input2_size"], 1, 1) + if self.input_tensors[1].constant(): + self.input_tensors[1].set_data(self.params["input2"]) self._add_output(self.params["output_idx"], self.params["output_dtype"], self.params["input_size"], 1, 1) if None in default_params: From 649354b780a76e7f39454339d59e58a3e78f3ea5 Mon Sep 17 00:00:00 2001 From: meenchen Date: Fri, 24 Feb 2023 14:06:15 -0500 Subject: [PATCH 10/19] fix test --- code_generator/TTEParser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/code_generator/TTEParser.py b/code_generator/TTEParser.py index 60a54efd..64e8a94a 100644 --- a/code_generator/TTEParser.py +++ b/code_generator/TTEParser.py @@ -1405,6 +1405,7 @@ def _convert_sub(self, op): "input2_idx": input2_info["name"], "output_idx": output_info["name"], "input_size": input0_h * input0_w * input0_c, + "input2_size": input0_h * input0_w * input0_c, "input_dtype": input_dtype, "input2_dtype": input2_dtype, "output_dtype": output_dtype, From e0635c0a4ea27a0bb14905041cc8861a750d50bf Mon Sep 17 00:00:00 2001 From: meenchen Date: Fri, 24 Feb 2023 14:18:00 -0500 Subject: [PATCH 11/19] debug info --- code_generator/CodeGenerator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code_generator/CodeGenerator.py b/code_generator/CodeGenerator.py index 9fd1b687..1bdcf14a 100644 --- a/code_generator/CodeGenerator.py +++ b/code_generator/CodeGenerator.py @@ -777,7 +777,7 @@ def _parseTrainable(self): for t in op.input_tensors: if t.constant(): if t.data is None: - raise ValueError("constant tensor data not found") + raise ValueError(f"constant tensor data not found for op:{layer_info['op']}") self._parseConstant(tensor) def _parseConstant(self, t: tensor): From 158c54c6fcc9f537561d3fe668a2512aaf96cec8 Mon Sep 17 00:00:00 2001 From: meenchen Date: Fri, 24 Feb 2023 14:28:44 -0500 Subject: [PATCH 12/19] TTE test --- code_generator/operators/mul.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/code_generator/operators/mul.py b/code_generator/operators/mul.py index 0a3fad49..da144cf5 100644 --- a/code_generator/operators/mul.py +++ b/code_generator/operators/mul.py @@ -46,7 +46,9 @@ def __init__(self, params: dict) -> None: if not (isParamstr(self.params["input2_idx"]) or islabelstr(self.params["input2_idx"])): self._add_input(self.params["input2_idx"], self.params["input2_dtype"], self.params["input2_size"], 1, 1) # TODO: Refactor this - if self.input_tensors[1].constant(): + if ( + self.input_tensors[1].constant() and self.params["constant"] is not None + ): # the last condition is for TTE self.input_tensors[1].set_data(self.params["input2"]) self._add_output(self.params["output_idx"], self.params["output_dtype"], self.params["output_size"], 1, 1) From 90c2c06fcbf58d48a166fe930a094bda06a2fcd3 Mon Sep 17 00:00:00 2001 From: meenchen Date: Fri, 24 Feb 2023 14:31:37 -0500 Subject: [PATCH 13/19] fix test --- code_generator/operators/mul.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/code_generator/operators/mul.py b/code_generator/operators/mul.py index da144cf5..6b9ac538 100644 --- a/code_generator/operators/mul.py +++ b/code_generator/operators/mul.py @@ -46,9 +46,7 @@ def __init__(self, params: dict) -> None: if not (isParamstr(self.params["input2_idx"]) or islabelstr(self.params["input2_idx"])): self._add_input(self.params["input2_idx"], self.params["input2_dtype"], self.params["input2_size"], 1, 1) # TODO: Refactor this - if ( - self.input_tensors[1].constant() and self.params["constant"] is not None - ): # the last condition is for TTE + if self.input_tensors[1].constant() and self.params["constant"] is None: # the last condition is for TTE self.input_tensors[1].set_data(self.params["input2"]) self._add_output(self.params["output_idx"], self.params["output_dtype"], self.params["output_size"], 1, 1) From a5162cd0c6633516f70399c783119d5115d65cb9 Mon Sep 17 00:00:00 2001 From: meenchen Date: Fri, 24 Feb 2023 14:38:23 -0500 Subject: [PATCH 14/19] fix --- code_generator/CodeGenerator.py | 4 +++- code_generator/operators/mul.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/code_generator/CodeGenerator.py b/code_generator/CodeGenerator.py index 1bdcf14a..e0aadefb 100644 --- a/code_generator/CodeGenerator.py +++ b/code_generator/CodeGenerator.py @@ -775,7 +775,9 @@ def _parseTrainable(self): else: # Parse constants of inputs for t in op.input_tensors: - if t.constant(): + if ( + t.constant() and "constant" in layer_info and layer_info["constant"] is None + ): # the last condition is for TTE if t.data is None: raise ValueError(f"constant tensor data not found for op:{layer_info['op']}") self._parseConstant(tensor) diff --git a/code_generator/operators/mul.py b/code_generator/operators/mul.py index 6b9ac538..0a3fad49 100644 --- a/code_generator/operators/mul.py +++ b/code_generator/operators/mul.py @@ -46,7 +46,7 @@ def __init__(self, params: dict) -> None: if not (isParamstr(self.params["input2_idx"]) or islabelstr(self.params["input2_idx"])): self._add_input(self.params["input2_idx"], self.params["input2_dtype"], self.params["input2_size"], 1, 1) # TODO: Refactor this - if self.input_tensors[1].constant() and self.params["constant"] is None: # the last condition is for TTE + if self.input_tensors[1].constant(): self.input_tensors[1].set_data(self.params["input2"]) self._add_output(self.params["output_idx"], self.params["output_dtype"], self.params["output_size"], 1, 1) From 4a56f14a31b7046595e19172aa41cfae5218d903 Mon Sep 17 00:00:00 2001 From: meenchen Date: Fri, 24 Feb 2023 16:25:08 -0500 Subject: [PATCH 15/19] fp weight --- code_generator/CodeGenerator.py | 70 ++++++++++-------- .../converters/tflite_parser/add.py | 6 +- .../converters/tflite_parser/concat.py | 2 +- code_generator/operators/add.py | 42 ++++++++--- code_generator/operators/basic_utils.py | 6 +- code_generator/operators/concat.py | 3 + code_generator/operators/conv2d.py | 74 ++++--------------- code_generator/operators/div.py | 3 + code_generator/operators/mul.py | 2 +- code_generator/operators/notequal.py | 2 +- code_generator/operators/sub.py | 3 +- 11 files changed, 106 insertions(+), 107 deletions(-) diff --git a/code_generator/CodeGenerator.py b/code_generator/CodeGenerator.py index e0aadefb..ddf8e4b6 100644 --- a/code_generator/CodeGenerator.py +++ b/code_generator/CodeGenerator.py @@ -775,12 +775,13 @@ def _parseTrainable(self): else: # Parse constants of inputs for t in op.input_tensors: - if ( - t.constant() and "constant" in layer_info and layer_info["constant"] is None - ): # the last condition is for TTE + if t.constant(): + # for TTE compatible + if "constant" in layer_info and layer_info["constant"] is None: + continue if t.data is None: raise ValueError(f"constant tensor data not found for op:{layer_info['op']}") - self._parseConstant(tensor) + self._parseConstant(t) def _parseConstant(self, t: tensor): def type_to_c_type(type: str) -> str: @@ -796,7 +797,7 @@ def type_to_c_type(type: str) -> str: fp = self.header_handle # 8bit implementation - string = f"const {type_to_c_type(t.dtype)}" + "[" + str(np.prod(t.size)) + "] = {" + string = f"const {type_to_c_type(t.dtype)} {t.graph_idx}" + "[" + str(np.prod(t.size)) + "] = {" flat_data = t.data.flatten() for d in flat_data: string += f"{d}, " @@ -850,34 +851,43 @@ def _parseEffectivescales(self, Lindex, scales): def _parseWeight(self, Lindex, weight, weight_name=None, is_const=True): fp = self.header_handle const_str = "const " if is_const else "" - string = f"{const_str}unsigned char weight" + str(Lindex) + "[" + str(len(weight)) + "] = {" - fp.write(string) - for _, value in enumerate(weight): - value = int(value) - if value < 0: - value += 256 - fp.write(str(format(value, "#04x")) + ", ") - fp.write("};\n") - - if self.is_training: - string = f"{const_str}float weight_fp" + str(Lindex) + "[" + str(len(weight)) + "] = {" + if weight.dtype == "float32": + string = f"{const_str}unsigned char weight_fp" + str(Lindex) + "[" + str(len(weight)) + "] = {" + for _, value in enumerate(weight): + string += f"{value}, " + string += "};\n" + fp.write(string) + elif weight.dtype == "int8": + string = f"{const_str}unsigned char weight" + str(Lindex) + "[" + str(len(weight)) + "] = {" fp.write(string) - for _, w in enumerate(weight): - value = float(w) - fp.write(str(value) + ", ") + for _, value in enumerate(weight): + value = int(value) + if value < 0: + value += 256 + fp.write(str(format(value, "#04x")) + ", ") fp.write("};\n") - if weight_name is not None: - for r in self.trainSRAMTable: - if r.name == weight_name: - return - self.trainSRAMTable.append(tensorRecorder(weight_name, len(weight), "unknown")) - - if weight.dtype == "int8": - string = f"{const_str}unsigned char* {weight_name}=weight" + str(Lindex) + ";\n" - else: - raise NotImplementedError - fp.write(string) + if self.is_training: + string = f"{const_str}float weight_fp" + str(Lindex) + "[" + str(len(weight)) + "] = {" + fp.write(string) + for _, w in enumerate(weight): + value = float(w) + fp.write(str(value) + ", ") + fp.write("};\n") + + if weight_name is not None: + for r in self.trainSRAMTable: + if r.name == weight_name: + return + self.trainSRAMTable.append(tensorRecorder(weight_name, len(weight), "unknown")) + + if weight.dtype == "int8": + string = f"{const_str}unsigned char* {weight_name}=weight" + str(Lindex) + ";\n" + else: + raise NotImplementedError + fp.write(string) + else: + raise NotImplementedError def _parseWeightPartial(self, Lindex, weight, first_k_channel=None, weight_name=None, is_const=True): fp = self.header_handle diff --git a/code_generator/converters/tflite_parser/add.py b/code_generator/converters/tflite_parser/add.py index 9a491e23..ef1b0a59 100644 --- a/code_generator/converters/tflite_parser/add.py +++ b/code_generator/converters/tflite_parser/add.py @@ -59,7 +59,7 @@ def parse_add(op, model: Model.Model): input2_idx = input2_tensor.tensor_idx try: input2 = get_np_from_wrapper(input2_tensor) - input2_idx = "constant" + input2_idx + input2_idx = "constant" + str(input2_idx) except Exception: input2 = None @@ -123,7 +123,9 @@ def parse_add(op, model: Model.Model): "output_h": output_h, "output_w": output_w, "output_c": output_c, - "dtypte": input_type, + "input_dtype": input_type, + "input2_dtype": input_type2, + "output_dtype": output_type, # trainable parameters "input_zero_point": input_zero_point, "input2_zero_point": input2_zero_point, diff --git a/code_generator/converters/tflite_parser/concat.py b/code_generator/converters/tflite_parser/concat.py index af7a75fa..64f45ddc 100644 --- a/code_generator/converters/tflite_parser/concat.py +++ b/code_generator/converters/tflite_parser/concat.py @@ -46,7 +46,7 @@ def parse_concat(op, model: Model.Model): input2_idx = input2_tensor.tensor_idx try: input2 = get_np_from_wrapper(input2_tensor) - input2_idx = "constant" + input2_idx + input2_idx = "constant" + str(input2_idx) except Exception: input2 = None diff --git a/code_generator/operators/add.py b/code_generator/operators/add.py index ff2c913c..8f19510e 100644 --- a/code_generator/operators/add.py +++ b/code_generator/operators/add.py @@ -78,7 +78,7 @@ def __init__(self, params: dict) -> None: ) # TODO: Refactor this if self.input_tensors[1].constant(): - self.input_tensors[1].set_data(self.params["input2"]) + self.input_tensors[1].set_data(self.params["input2"], self.params["input2_idx"]) self._add_output( self.params["output_idx"], self.params["output_dtype"], @@ -116,15 +116,35 @@ def generate_inference_str(self): ) else: if isinstance(params["input2_idx"], str) and "constant" in params["input2_idx"]: - logging.warn("Add operator with constant support is still no ready.") + t = self.input_tensors[1] + assert t.data is not None + # elementwise add + if t.num_elements() == self.input_tensors[0].num_elements(): + string += ( + f"add_fp({str(int(params['input_h']*params['input_w']*params['input_c']))}, " + + f"{self._getBufferstr(params['input_buf_add'], params['input_buf_add_offset'])}," + + f"{t.graph_idx}," + + f"{self._getBufferstr(params['output_buf_add'], params['output_buf_add_offset'])});\n" + ) + # scaler or vector based + else: + logging.warn("Add operator with constant support is still no ready.") else: - string += ( - f"add_fpreq({str(int(params['input_h']*params['input_w']*params['input_c']))}, " - + f"{self._getBufferstr(params['input_buf_add'], params['input_buf_add_offset'])}," - + f"{str(params['input_scale'])},{str(params['input_zero_point'])}," - + f"{self._getBufferstr(params['input2_buf_add'], params['input2_buf_add_offset'])}," - + f"{str(params['input2_scale'])},{str(params['input2_zero_point'])}," - + f"{str(params['output_scale'])},{str(params['output_zero_point'])}," - + f"{self._getBufferstr(params['output_buf_add'], params['output_buf_add_offset'])});\n" - ) + if params["input_dtype"] == "int8": + string += ( + f"add_fpreq({str(int(params['input_h']*params['input_w']*params['input_c']))}, " + + f"{self._getBufferstr(params['input_buf_add'], params['input_buf_add_offset'])}," + + f"{str(params['input_scale'])},{str(params['input_zero_point'])}," + + f"{self._getBufferstr(params['input2_buf_add'], params['input2_buf_add_offset'])}," + + f"{str(params['input2_scale'])},{str(params['input2_zero_point'])}," + + f"{str(params['output_scale'])},{str(params['output_zero_point'])}," + + f"{self._getBufferstr(params['output_buf_add'], params['output_buf_add_offset'])});\n" + ) + elif params["input_dtype"] == "float32": + string += ( + f"add_fp({str(int(params['input_h']*params['input_w']*params['input_c']))}, " + + f"{self._getBufferstr(params['input_buf_add'], params['input_buf_add_offset'])}," + + f"{self._getBufferstr(params['input2_buf_add'], params['input2_buf_add_offset'])}," + + f"{self._getBufferstr(params['output_buf_add'], params['output_buf_add_offset'])});\n" + ) return string diff --git a/code_generator/operators/basic_utils.py b/code_generator/operators/basic_utils.py index ade40470..ddb446fc 100644 --- a/code_generator/operators/basic_utils.py +++ b/code_generator/operators/basic_utils.py @@ -230,8 +230,9 @@ def __init__(self, graph_idx, dtype, dims) -> None: else: self.is_constant = False - def set_data(self, data: np.ndarray): + def set_data(self, data: np.ndarray, name): self.data = data + self.buffer_name = name def constant(self): return self.is_constant @@ -251,6 +252,9 @@ def set_input_w(self, w): def set_input_h(self, h): self.size = (self.size[0], self.size[1], h) + def num_elements(self): + return np.prod(self.size) + def len(self): byte_cnt = math.ceil(np.prod(self.size) * self.byte_size[self.dtype]) # align the memory to 4 diff --git a/code_generator/operators/concat.py b/code_generator/operators/concat.py index a0240ef6..7b53ee26 100644 --- a/code_generator/operators/concat.py +++ b/code_generator/operators/concat.py @@ -51,6 +51,9 @@ def __init__(self, params: dict) -> None: self.params["input2_w"], self.params["input2_c"], ) + # TODO: Refactor this + if self.input_tensors[1].constant(): + self.input_tensors[1].set_data(self.params["input2"], self.params["input2_idx"]) self._add_output( self.params["output_idx"], self.params["output_dtype"], diff --git a/code_generator/operators/conv2d.py b/code_generator/operators/conv2d.py index a04e3a6e..addf28b5 100644 --- a/code_generator/operators/conv2d.py +++ b/code_generator/operators/conv2d.py @@ -149,73 +149,29 @@ def generate_inference_str( params = self.params # floating point implmenetation if params["input_dtype"] == params["output_dtype"] == "float32": - string += f"conv_params.stride_height = {params['stride_h']};\n" - string += f"conv_params.stride_width = {params['stride_w']};\n" - string += f"conv_params.dilation_width_factor = {params['dilation_w']};\n" - string += f"conv_params.dilation_height_factor = {params['dilation_h']};\n" - string += f"conv_params.input_offset = {params['input_zero_point']};\n" - string += f"conv_params.output_offset = {params['output_zero_point']};\n" - string += f"conv_params.padding_values.width = {params['padding_w']};\n" - string += f"conv_params.padding_values.height = {params['padding_h']};\n" - string += "conv_params.quantized_activation_min = -128;\n" - string += "conv_params.quantized_activation_max = 127;\n" - string += f"conv_params.float_activation_min = {params['float_min']};\n" - string += f"conv_params.float_activation_max = {params['float_max']};\n" - if isinstance(params["weight_name"], str) and isweightstr(params["weight_name"]): weight_string = params["weight_name"] else: weight_string = f"weight_fp{params['parsed_trainable']}" + assert params["stride_h"] == params["stride_w"] + stride = params["stride_h"] + input = f"{self._getBufferstrCast(params['input_buf_add'], params['input_buf_add_offset'])}" + output = f"{self._getBufferstrCast(params['output_buf_add'], params['output_buf_add_offset'])}" + output_h = f"{str(params['output_h'])}" + output_w = f"{str(params['output_w'])}" + output_c = f"{str(params['output_c'])}" + input_h = f"{str(params['input_h'])}" + input_w = f"{str(params['input_w'])}" + input_c = f"{str(params['input_c'])}" + kernel_h = f"{str(params['kernel_h'])}" + kernel_w = f"{str(params['kernel_w'])}" string += ( - "TFLite_Conv_fp(conv_params," - + f"{self._getBufferstrCast(params['input_buf_add'], params['input_buf_add_offset'])}," - + f"{params['input_h']},{params['input_w']},{params['input_c']}," - + f"{weight_string},{params['kernel_h']},{params['kernel_w']},{params['input_c']},NULL," - + f"{self._getBufferstrCast(params['output_buf_add'], params['output_buf_add_offset'])}," - + f"{str(params['output_h'])},{str(params['output_w'])},{str(params['output_c'])},(float*)sbuf,1);\n" + f"conv_fp({input}, {input_h}, {input_w}, {input_c}, {stride}, {kernel_h}, {kernel_w}, " + + f"{weight_string}, {output}, {output_h}, {output_w}, {output_c},(float*)sbuf);" ) elif params["input_dtype"] == params["output_dtype"] == "int8" and tflite_op and (not USE_TTE_INT8): - string += f"conv_params.stride_height = {params['stride_h']};\n" - string += f"conv_params.stride_width = {params['stride_w']};\n" - string += "conv_params.dilation_width_factor = 1;\n" - string += "conv_params.dilation_height_factor = 1;\n" - string += f"conv_params.input_offset = {params['input_zero_point']};\n" - string += f"conv_params.output_offset = {params['output_zero_point']};\n" - string += f"conv_params.padding_values.width = {params['padding_w']};\n" - string += f"conv_params.padding_values.height = {params['padding_h']};\n" - string += "conv_params.quantized_activation_min = -128;\n" - string += "conv_params.quantized_activation_max = 127;\n" - string += f"conv_params.float_activation_min = {params['float_min']};\n" - string += f"conv_params.float_activation_max = {params['float_max']};\n" - - parsed_idx = str(params["parsed_trainable"]) - - function_name = "TFLite_Conv_int8_PerChannel" - if params["first_k_channel"] is not None: # partial channels in SRAM, - function_name += "_partialCH" - weight_string = ( - f"(const q7_t*)weight{parsed_idx},(const q7_t*)weight{parsed_idx}Flash,{params['first_k_channel']}" - ) - else: - weight_string = f"(const q7_t*) weight{parsed_idx}" - - if dummy_address: - input_address_string = "&buffer0[0]" - output_address_string = "&buffer0[0]" - else: - input_address_string = f"{self._getBufferstr(params['input_buf_add'], params['input_buf_add_offset'])}" - output_address_string = ( - f"{self._getBufferstr(params['output_buf_add'], params['output_buf_add_offset'])}" - ) - - string += ( - f"{function_name}(conv_params,multiplier{parsed_idx},shift{parsed_idx}," - + f"{params['input_h']},{params['input_w']},{params['input_c']},{input_address_string}," - + f"{weight_string},{params['kernel_h']},{params['kernel_w']},{params['input_c']},bias{parsed_idx}," - + f"{str(params['output_h'])},{str(params['output_w'])},{str(params['output_c'])}," - + f"{output_address_string},1);\n" - ) + raise NotImplementedError else: kernel_h = params["kernel_h"] # function name diff --git a/code_generator/operators/div.py b/code_generator/operators/div.py index 0378bd7c..1fe17432 100644 --- a/code_generator/operators/div.py +++ b/code_generator/operators/div.py @@ -52,6 +52,9 @@ def __init__(self, params: dict) -> None: self._add_input(self.params["input_idx"], self.params["input_dtype"], self.params["input_size"], 1, 1) if "constant" not in self.params["input2_idx"]: self._add_input(self.params["input2_idx"], self.params["input2_dtype"], self.params["input_size"], 1, 1) + # TODO: Refactor this + if self.input_tensors[1].constant(): + self.input_tensors[1].set_data(self.params["input2"], self.params["input2_idx"]) self._add_output(self.params["output_idx"], self.params["output_dtype"], self.params["input_size"], 1, 1) if None in default_params: diff --git a/code_generator/operators/mul.py b/code_generator/operators/mul.py index 0a3fad49..85d61c02 100644 --- a/code_generator/operators/mul.py +++ b/code_generator/operators/mul.py @@ -47,7 +47,7 @@ def __init__(self, params: dict) -> None: self._add_input(self.params["input2_idx"], self.params["input2_dtype"], self.params["input2_size"], 1, 1) # TODO: Refactor this if self.input_tensors[1].constant(): - self.input_tensors[1].set_data(self.params["input2"]) + self.input_tensors[1].set_data(self.params["input2"], self.params["input2_idx"]) self._add_output(self.params["output_idx"], self.params["output_dtype"], self.params["output_size"], 1, 1) if None in default_params: diff --git a/code_generator/operators/notequal.py b/code_generator/operators/notequal.py index 3129c1fc..401e12f8 100644 --- a/code_generator/operators/notequal.py +++ b/code_generator/operators/notequal.py @@ -50,7 +50,7 @@ def __init__(self, params: dict) -> None: ) # TODO: Refactor this if self.input_tensors[1].constant(): - self.input_tensors[1].set_data(self.params["input2"]) + self.input_tensors[1].set_data(self.params["input2"], self.params["input2_idx"]) self._add_output( self.params["output_idx"], self.params["output_dtype"], diff --git a/code_generator/operators/sub.py b/code_generator/operators/sub.py index d1ebfd6e..f6ae0937 100644 --- a/code_generator/operators/sub.py +++ b/code_generator/operators/sub.py @@ -38,8 +38,9 @@ def __init__(self, params: dict) -> None: # handle input/output tensors in HWC format self._add_input(self.params["input_idx"], self.params["input_dtype"], self.params["input_size"], 1, 1) self._add_input(self.params["input2_idx"], self.params["input2_dtype"], self.params["input2_size"], 1, 1) + # TODO: Refactor this if self.input_tensors[1].constant(): - self.input_tensors[1].set_data(self.params["input2"]) + self.input_tensors[1].set_data(self.params["input2"], self.params["input2_idx"]) self._add_output(self.params["output_idx"], self.params["output_dtype"], self.params["input_size"], 1, 1) if None in default_params: From 5e4bb87e0f906754b80e0c934919e4d320dfafdb Mon Sep 17 00:00:00 2001 From: meenchen Date: Fri, 24 Feb 2023 16:43:08 -0500 Subject: [PATCH 16/19] fix --- code_generator/CodeGenerator.py | 2 +- code_generator/operators/conv2d.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/code_generator/CodeGenerator.py b/code_generator/CodeGenerator.py index ddf8e4b6..6ab1c1b0 100644 --- a/code_generator/CodeGenerator.py +++ b/code_generator/CodeGenerator.py @@ -777,7 +777,7 @@ def _parseTrainable(self): for t in op.input_tensors: if t.constant(): # for TTE compatible - if "constant" in layer_info and layer_info["constant"] is None: + if "constant" in layer_info and layer_info["constant"] is not None: continue if t.data is None: raise ValueError(f"constant tensor data not found for op:{layer_info['op']}") diff --git a/code_generator/operators/conv2d.py b/code_generator/operators/conv2d.py index addf28b5..361dea3b 100644 --- a/code_generator/operators/conv2d.py +++ b/code_generator/operators/conv2d.py @@ -168,7 +168,7 @@ def generate_inference_str( string += ( f"conv_fp({input}, {input_h}, {input_w}, {input_c}, {stride}, {kernel_h}, {kernel_w}, " - + f"{weight_string}, {output}, {output_h}, {output_w}, {output_c},(float*)sbuf);" + + f"{weight_string}, {output}, {output_h}, {output_w}, {output_c},(float*)sbuf);\n" ) elif params["input_dtype"] == params["output_dtype"] == "int8" and tflite_op and (not USE_TTE_INT8): raise NotImplementedError From 4bcdcc34f841743c56f6a6eaa88200c601c27a88 Mon Sep 17 00:00:00 2001 From: meenchen Date: Fri, 24 Feb 2023 17:52:41 -0500 Subject: [PATCH 17/19] gen most op (mean still on-going) --- code_generator/operators/add.py | 8 +----- code_generator/operators/batchmatmul.py | 20 ++++++++++----- code_generator/operators/conv2d.py | 9 ++----- code_generator/operators/equal.py | 32 ++++++++++++++++++----- code_generator/operators/notequal.py | 32 ++++++++++++++++++----- code_generator/operators/rsqrt.py | 15 +++++------ code_generator/operators/slice.py | 25 +++++++++++++++--- code_generator/operators/softmax.py | 20 +++++++-------- code_generator/operators/squarddiff.py | 34 ++++++++++++++++++------- code_generator/operators/sub.py | 30 +++++++++++++++------- 10 files changed, 150 insertions(+), 75 deletions(-) diff --git a/code_generator/operators/add.py b/code_generator/operators/add.py index 8f19510e..662000e3 100644 --- a/code_generator/operators/add.py +++ b/code_generator/operators/add.py @@ -1,6 +1,3 @@ -import logging -import warnings - from ..constant import USE_BIT_MASK from .basic_utils import basicOperator, deep_copy_dicts, overwrite_dicts @@ -87,9 +84,6 @@ def __init__(self, params: dict) -> None: self.params["output_h"], ) - if None in default_params: - warnings.warn(f"parameters are not all set for op {self.params['op']}") - def get_macs(self) -> int: p = self.params return p["output_h"] * p["output_w"] * p["output_c"] @@ -128,7 +122,7 @@ def generate_inference_str(self): ) # scaler or vector based else: - logging.warn("Add operator with constant support is still no ready.") + raise NotImplementedError("add with scaler/vector constant support is still no ready.") else: if params["input_dtype"] == "int8": string += ( diff --git a/code_generator/operators/batchmatmul.py b/code_generator/operators/batchmatmul.py index 57d9f8ed..6dbaee37 100644 --- a/code_generator/operators/batchmatmul.py +++ b/code_generator/operators/batchmatmul.py @@ -1,6 +1,3 @@ -import logging -import warnings - from .basic_utils import basicOperator, deep_copy_dicts, overwrite_dicts __all__ = ["batchmatmul"] @@ -56,15 +53,24 @@ def __init__(self, params: dict) -> None: self.params["output_col"], ) - if None in default_params: - warnings.warn(f"parameters are not all set for op {self.params['op']}") - def generate_inference_str(self): params = self.params string = "" + input = f"{self._getBufferstrCast(params['input_buf_add'], params['input_buf_add_offset'])}" + input_col = self.params["input_col"] + input_row = self.params["input_row"] + input2 = f"{self._getBufferstrCast(params['input_buf_add'], params['input_buf_add_offset'])}" + input2_col = self.params["input2_col"] + input2_row = self.params["input2_row"] + output = f"{self._getBufferstrCast(params['output_buf_add'], params['output_buf_add_offset'])}" + output_col = self.params["output_col"] + output_row = self.params["output_row"] if params["input_dtype"] == "float32": - logging.warn("BATCHMAMUL still needs implementation.") + string += ( + f"batchmatmul_fp({params['batch_size']},{input},{input_col},{input_row}," + + f"{input2},{input2_col},{input2_row},{output},{output_col},{output_row});\n" + ) else: raise NotImplementedError diff --git a/code_generator/operators/conv2d.py b/code_generator/operators/conv2d.py index 361dea3b..027a4904 100644 --- a/code_generator/operators/conv2d.py +++ b/code_generator/operators/conv2d.py @@ -1,5 +1,3 @@ -import warnings - from ..constant import USE_BIT_MASK, USE_TTE_INT8 from .basic_utils import basicOperator, deep_copy_dicts, isweightstr, overwrite_dicts @@ -82,9 +80,6 @@ def __init__(self, params: dict) -> None: self.params["output_h"], ) - if None in default_params: - warnings.warn(f"parameters are not all set for op {self.params['op']}") - def _op_hparam_info(self) -> str: return ( f" k{self.params['kernel_h']}x{self.params['kernel_w']}_r{self.params['input_h']}x" @@ -167,8 +162,8 @@ def generate_inference_str( kernel_w = f"{str(params['kernel_w'])}" string += ( - f"conv_fp({input}, {input_h}, {input_w}, {input_c}, {stride}, {kernel_h}, {kernel_w}, " - + f"{weight_string}, {output}, {output_h}, {output_w}, {output_c},(float*)sbuf);\n" + f"conv_fp({input},{input_h},{input_w},{input_c},{stride},{kernel_h},{kernel_w}," + + f"{weight_string},{output},{output_h},{output_w},{output_c},(float*)sbuf);\n" ) elif params["input_dtype"] == params["output_dtype"] == "int8" and tflite_op and (not USE_TTE_INT8): raise NotImplementedError diff --git a/code_generator/operators/equal.py b/code_generator/operators/equal.py index 1aa25e78..44708022 100644 --- a/code_generator/operators/equal.py +++ b/code_generator/operators/equal.py @@ -1,6 +1,3 @@ -import logging -import warnings - from .basic_utils import basicOperator, deep_copy_dicts, overwrite_dicts __all__ = ["equal"] @@ -56,15 +53,36 @@ def __init__(self, params: dict) -> None: self.params["output_c"], ) - if None in default_params: - warnings.warn(f"parameters are not all set for op {self.params['op']}") - def generate_inference_str(self): params = self.params string = "" + input = f"{self._getBufferstrCast(params['input_buf_add'], params['input_buf_add_offset'])}" + input_h = f"{str(params['input_h'])}" + input_w = f"{str(params['input_w'])}" + input_c = f"{str(params['input_c'])}" + if self.input_tensors[1].constant(): + input2 = f"{self.input_tensors[1].graph_idx}" + else: + input2 = f"{self._getBufferstrCast(params['input2_buf_add'], params['input2_buf_add_offset'])}" + input2_h = f"{str(params['input2_h'])}" + input2_w = f"{str(params['input2_w'])}" + input2_c = f"{str(params['input2_c'])}" + output = f"{self._getBufferstrCast(params['output_buf_add'], params['output_buf_add_offset'])}" + output_h = f"{str(params['output_h'])}" + output_w = f"{str(params['output_w'])}" + output_c = f"{str(params['output_c'])}" if params["input_dtype"] in ["float32", "int32", "bool"]: - logging.warn("EQUAL still needs implementation.") + if params["input_dtype"] == "float32": + function_name = "equal_fp" + elif params["input_dtype"] == "int32": + function_name = "equal_int32" + elif params["input_dtype"] == "bool": + function_name = "equal_bool" + string += ( + f"{function_name}({input},{input_h},{input_w},{input_c}," + + f"{input2},{input2_h},{input2_w},{input2_c},{output},{output_h},{output_w},{output_c});\n" + ) else: raise NotImplementedError diff --git a/code_generator/operators/notequal.py b/code_generator/operators/notequal.py index 401e12f8..70ff285b 100644 --- a/code_generator/operators/notequal.py +++ b/code_generator/operators/notequal.py @@ -1,6 +1,3 @@ -import logging -import warnings - from .basic_utils import basicOperator, deep_copy_dicts, overwrite_dicts __all__ = ["notequal"] @@ -59,15 +56,36 @@ def __init__(self, params: dict) -> None: self.params["output_c"], ) - if None in default_params: - warnings.warn(f"parameters are not all set for op {self.params['op']}") - def generate_inference_str(self): params = self.params string = "" + input = f"{self._getBufferstrCast(params['input_buf_add'], params['input_buf_add_offset'])}" + input_h = f"{str(params['input_h'])}" + input_w = f"{str(params['input_w'])}" + input_c = f"{str(params['input_c'])}" + if self.input_tensors[1].constant(): + input2 = f"{self.input_tensors[1].graph_idx}" + else: + input2 = f"{self._getBufferstrCast(params['input2_buf_add'], params['input2_buf_add_offset'])}" + input2_h = f"{str(params['input2_h'])}" + input2_w = f"{str(params['input2_w'])}" + input2_c = f"{str(params['input2_c'])}" + output = f"{self._getBufferstrCast(params['output_buf_add'], params['output_buf_add_offset'])}" + output_h = f"{str(params['output_h'])}" + output_w = f"{str(params['output_w'])}" + output_c = f"{str(params['output_c'])}" if params["input_dtype"] in ["float32", "int32", "bool"]: - logging.warn("NOT_EQUAL still needs implementation.") + if params["input_dtype"] == "float32": + function_name = "notequal_fp" + elif params["input_dtype"] == "int32": + function_name = "notequal_int32" + elif params["input_dtype"] == "bool": + function_name = "notequal_bool" + string += ( + f"{function_name}({input},{input_h},{input_w},{input_c}," + + f"{input2},{input2_h},{input2_w},{input2_c},{output},{output_h},{output_w},{output_c});\n" + ) else: raise NotImplementedError diff --git a/code_generator/operators/rsqrt.py b/code_generator/operators/rsqrt.py index 16717902..5d5afb7a 100644 --- a/code_generator/operators/rsqrt.py +++ b/code_generator/operators/rsqrt.py @@ -1,6 +1,3 @@ -import logging -import warnings - from .basic_utils import basicOperator, deep_copy_dicts, overwrite_dicts __all__ = ["rsqrt"] @@ -40,13 +37,15 @@ def __init__(self, params: dict) -> None: 1, ) - if None in default_params: - warnings.warn(f"parameters are not all set for op {self.params['op']}") - def generate_inference_str(self): - # params = self.params + params = self.params string = "" + input = f"{self._getBufferstrCast(params['input_buf_add'], params['input_buf_add_offset'])}" + output = f"{self._getBufferstrCast(params['output_buf_add'], params['output_buf_add_offset'])}" - logging.warn("RSQRT operator support is still no ready.") + if params["input_dtype"] == "float32": + string += f"rsqrt_fp({self.input_tensors[0].num_elements()},{input},{output});\n" + else: + raise NotImplementedError return string diff --git a/code_generator/operators/slice.py b/code_generator/operators/slice.py index 5aabb043..a9ceb958 100644 --- a/code_generator/operators/slice.py +++ b/code_generator/operators/slice.py @@ -1,5 +1,3 @@ -import logging - from .basic_utils import basicOperator, deep_copy_dicts, overwrite_dicts __all__ = ["slice"] @@ -51,9 +49,28 @@ def __init__(self, params: dict) -> None: ) def generate_inference_str(self): - # params = self.params + params = self.params string = "" + input = f"{self._getBufferstrCast(params['input_buf_add'], params['input_buf_add_offset'])}" + input_h = f"{str(params['input_h'])}" + input_w = f"{str(params['input_w'])}" + input_c = f"{str(params['input_c'])}" + begin_h = f"{str(params['begin_h'])}" + begin_w = f"{str(params['begin_w'])}" + begin_c = f"{str(params['begin_c'])}" + output = f"{self._getBufferstrCast(params['output_buf_add'], params['output_buf_add_offset'])}" + output_h = f"{str(params['output_h'])}" + output_w = f"{str(params['output_w'])}" + output_c = f"{str(params['output_c'])}" - logging.warn("slice operator is still no ready.") + if params["input_dtype"] == "float32": + if params["input_dtype"] == "float32": + function_name = "slice3d_fp" + string += ( + f"{function_name}({input},{input_h},{input_w},{input_c}," + + f"{begin_h},{begin_w},{begin_c},{output},{output_h},{output_w},{output_c});\n" + ) + else: + raise NotImplementedError return string diff --git a/code_generator/operators/softmax.py b/code_generator/operators/softmax.py index 177584c1..89f698a7 100644 --- a/code_generator/operators/softmax.py +++ b/code_generator/operators/softmax.py @@ -16,9 +16,6 @@ # Target ISA: ARMv7E-M # ---------------------------------------------------------------------- -import logging -import warnings - from .basic_utils import basicOperator, deep_copy_dicts, overwrite_dicts __all__ = ["softmax"] @@ -38,8 +35,8 @@ "output_h": None, "output_w": None, "output_c": None, - "input_dtype": "int8", - "output_dtype": "int8", + "input_dtype": None, + "output_dtype": None, # quantization related "input_zero_point": None, "output_zero_point": None, @@ -68,14 +65,17 @@ def __init__(self, params: dict) -> None: self.params["output_w"], self.params["output_c"], ) - - if None in default_params: - warnings.warn(f"parameters are not all set for op {self.params['op']}") + assert self.input_tensors[0].num_elements() == self.output_tensors[0].num_elements() def generate_inference_str(self): - # params = self.params + params = self.params string = "" + input = f"{self._getBufferstrCast(params['input_buf_add'], params['input_buf_add_offset'])}" + output = f"{self._getBufferstrCast(params['output_buf_add'], params['output_buf_add_offset'])}" - logging.warn("Add operator with constant support is still no ready.") + if params["input_dtype"] == "float32": + string += f"softmax_fp({self.input_tensors[0].num_elements()},{input},{output});\n" + else: + raise NotImplementedError return string diff --git a/code_generator/operators/squarddiff.py b/code_generator/operators/squarddiff.py index cdf7ff68..8c611e30 100644 --- a/code_generator/operators/squarddiff.py +++ b/code_generator/operators/squarddiff.py @@ -1,6 +1,3 @@ -import logging -import warnings - from .basic_utils import basicOperator, deep_copy_dicts, overwrite_dicts __all__ = ["squarddiff"] @@ -44,7 +41,7 @@ def __init__(self, params: dict) -> None: self.params["input_w"], self.params["input_c"], ) - if isinstance(self.params["input2_idx"], str) and "constant" not in self.params["input2_idx"]: + if not (isinstance(self.params["input2_idx"], str) and "constant" not in self.params["input2_idx"]): self._add_input( self.params["input2_idx"], self.params["input2_dtype"], @@ -52,6 +49,9 @@ def __init__(self, params: dict) -> None: self.params["input2_w"], self.params["input2_c"], ) + # TODO: Refactor this + if self.input_tensors[1].constant(): + self.input_tensors[1].set_data(self.params["input2"], self.params["input2_idx"]) self._add_output( self.params["output_idx"], self.params["output_dtype"], @@ -60,16 +60,32 @@ def __init__(self, params: dict) -> None: self.params["output_c"], ) - if None in default_params: - warnings.warn(f"parameters are not all set for op {self.params['op']}") - def generate_inference_str(self): params = self.params string = "" + input = f"{self._getBufferstrCast(params['input_buf_add'], params['input_buf_add_offset'])}" + input_h = f"{str(params['input_h'])}" + input_w = f"{str(params['input_w'])}" + input_c = f"{str(params['input_c'])}" + if self.input_tensors[1].constant(): + input2 = f"{self.input_tensors[1].graph_idx}" + else: + input2 = f"{self._getBufferstrCast(params['input2_buf_add'], params['input2_buf_add_offset'])}" + input2_h = f"{str(params['input2_h'])}" + input2_w = f"{str(params['input2_w'])}" + input2_c = f"{str(params['input2_c'])}" + output = f"{self._getBufferstrCast(params['output_buf_add'], params['output_buf_add_offset'])}" + output_h = f"{str(params['output_h'])}" + output_w = f"{str(params['output_w'])}" + output_c = f"{str(params['output_c'])}" if params["input_dtype"] == "float32": - logging.warn("squarddiff operator with constant support is still no ready.") + if params["input_dtype"] == "float32": + function_name = "squarddiff_fp" + string += ( + f"{function_name}({input},{input_h},{input_w},{input_c}," + + f"{input2},{input2_h},{input2_w},{input2_c},{output},{output_h},{output_w},{output_c});\n" + ) else: raise NotImplementedError - return string diff --git a/code_generator/operators/sub.py b/code_generator/operators/sub.py index f6ae0937..3381cfb7 100644 --- a/code_generator/operators/sub.py +++ b/code_generator/operators/sub.py @@ -1,6 +1,3 @@ -import logging -import warnings - from .basic_utils import basicOperator, deep_copy_dicts, overwrite_dicts __all__ = ["sub"] @@ -43,15 +40,30 @@ def __init__(self, params: dict) -> None: self.input_tensors[1].set_data(self.params["input2"], self.params["input2_idx"]) self._add_output(self.params["output_idx"], self.params["output_dtype"], self.params["input_size"], 1, 1) - if None in default_params: - warnings.warn(f"parameters are not all set for op {self.params['op']}") - def generate_inference_str(self): params = self.params - + string = "" if isinstance(params["input2_idx"], str) and "constant" in params["input2_idx"]: - logging.warn("Please implement sub operator with a constant tensor.") - + t = self.input_tensors[1] + assert t.data is not None + # elementwise add + if t.num_elements() == self.input_tensors[0].num_elements(): + string += ( + f"sub_fp({str(int(params['input_size']))}," + + f"{self._getBufferstr(params['input_buf_add'], params['input_buf_add_offset'])}," + + f"{t.graph_idx}," + + f"{self._getBufferstr(params['output_buf_add'], params['output_buf_add_offset'])});\n" + ) + # scaler or vector based + elif t.num_elements() == 1: + string += ( + f"sub_scaler_fp({str(int(params['input_size']))}," + + f"{self._getBufferstr(params['input_buf_add'], params['input_buf_add_offset'])}," + + f"{t.graph_idx}," + + f"{self._getBufferstr(params['output_buf_add'], params['output_buf_add_offset'])});\n" + ) + else: + raise NotImplementedError("sub with scaler/vector constant support is still no ready.") if params["input_dtype"] == "float32": string = ( f"sub({self.params['input_size']}," From 5aebf949248b5b2077917d018d7a91376ddf561e Mon Sep 17 00:00:00 2001 From: meenchen Date: Mon, 27 Feb 2023 16:02:20 -0500 Subject: [PATCH 18/19] fusion rule --- code_generator/TfliteConvertor.py | 89 +++++++++++-------- .../converters/tflite_parser/__init__.py | 2 +- .../converters/tflite_parser/mean1dto2d.py | 2 +- 3 files changed, 52 insertions(+), 41 deletions(-) diff --git a/code_generator/TfliteConvertor.py b/code_generator/TfliteConvertor.py index db38ab75..9c779289 100644 --- a/code_generator/TfliteConvertor.py +++ b/code_generator/TfliteConvertor.py @@ -27,6 +27,26 @@ from .constant import SKIP_OPs from .tflite import Model +regular_opconverter = { + "ADD": TF_Parser.parse_add, + "AVERAGE_POOL_2D": TF_Parser.parse_avgpool, + "RESIZE_NEAREST_NEIGHBOR": TF_Parser.parse_upsample, + "MAX_POOL_2D": TF_Parser.parse_maxpool, + "FULLY_CONNECTED": TF_Parser.parse_fc, + "DIV": TF_Parser.parse_div, + "BATCH_MATMUL": TF_Parser.parse_batchmatmul, + "NOT_EQUAL": TF_Parser.parse_notequal, + "EQUAL": TF_Parser.parse_equal, + "CONCATENATION": TF_Parser.parse_notequal, + "CAST": TF_Parser.parse_cast, + "SUB": TF_Parser.parse_sub, + "MUL": TF_Parser.parse_mul, + "SOFTMAX": TF_Parser.parse_softmax, + "SQUARED_DIFFERENCE": TF_Parser.parse_squarddiff, + "RSQRT": TF_Parser.parse_rsqrt, + "SLICE": TF_Parser.parse_slice, +} + # Parse tflite model into TinyEngine IR format class TfliteConvertor(object): @@ -107,6 +127,22 @@ def parseOperatorInfo(self): self.layer.append(SEelementmult_op) continue + if i + 1 < operators_len - 1: + next_op = self.subgraph.Operators(i + 1) + two_op_sequence = [op, next_op] + + if self.checIfMergeTwoMean1d(two_op_sequence): + logging.info("found target to merge two mean1d") + skip_next_ops = 1 + ret_op = TF_Parser.parse_mean1dto2d(op, self.model, self.average_1D_to_2D_holder) + ret_op = TF_Parser.parse_mean1dto2d(next_op, self.model, self.average_1D_to_2D_holder) + if ret_op is not None: + if self.skip_transpose is not None: + ret_op.params["input_idx"] = self.skip_transpose.input_idx + ret_op.input_tensors[0].graph_idx = self.skip_transpose.input_idx + self.layer.append(ret_op) + + continue # parse the op self._handleOperator(op) @@ -120,57 +156,21 @@ def _handleOperator(self, op): if op_code_str == "CONV_2D": self.layer.append(TF_Parser.parse_conv2d(op, self.model, self.tmpPADIndice)) self.tmpPADIndice = None - elif op_code_str == "ADD": - self.layer.append(TF_Parser.parse_add(op, self.model)) - elif op_code_str == "AVERAGE_POOL_2D": - self.layer.append(TF_Parser.parse_avgpool(op, self.model)) elif op_code_str == "DEPTHWISE_CONV_2D": self.layer.append(TF_Parser.parse_conv2d(op, self.model, self.tmpPADIndice)) self.tmpPADIndice = None elif op_code_str == "PAD": self._convert_PAD(op) - elif op_code_str == "RESIZE_NEAREST_NEIGHBOR": - self.layer.append(TF_Parser.parse_upsample(op, self.model)) - elif op_code_str == "MAX_POOL_2D": - self.layer.append(TF_Parser.parse_maxpool(op, self.model)) elif op_code_str in "MEAN": - ret_op = TF_Parser.parse_mead1dto2d(op, self.model, self.average_1D_to_2D_holder) - if ret_op is not None: - # TODO: This only handle a specific graph: TRANSPOSE -> MEAN -> MEANS - if self.skip_transpose is not None: - ret_op.params["input_idx"] = self.skip_transpose.input_idx - ret_op.input_tensors[0].graph_idx = self.skip_transpose.input_idx - self.layer.append(ret_op) + pass elif op_code_str == "TRANSPOSE": self._convert_TRANSPOSE(op) elif op_code_str == "FULLY_CONNECTED": self.layer.append(TF_Parser.parse_fc(op, self.model)) elif op_code_str == "RESHAPE": self.inplace_reshape_table.append(TF_Parser.parse_reshape_fuse_tensor_tuple(op, self.model)) - elif op_code_str == "DIV": - self.layer.append(TF_Parser.parse_div(op, self.model)) - elif op_code_str == "BATCH_MATMUL": - self.layer.append(TF_Parser.parse_batchmatmul(op, self.model)) - elif op_code_str == "NOT_EQUAL": - self.layer.append(TF_Parser.parse_notequal(op, self.model)) - elif op_code_str == "EQUAL": - self.layer.append(TF_Parser.parse_equal(op, self.model)) - elif op_code_str == "CONCATENATION": - self.layer.append(TF_Parser.parse_notequal(op, self.model)) - elif op_code_str == "CAST": - self.layer.append(TF_Parser.parse_cast(op, self.model)) - elif op_code_str == "SUB": - self.layer.append(TF_Parser.parse_sub(op, self.model)) - elif op_code_str == "MUL": - self.layer.append(TF_Parser.parse_mul(op, self.model)) - elif op_code_str == "SOFTMAX": - self.layer.append(TF_Parser.parse_softmax(op, self.model)) - elif op_code_str == "SQUARED_DIFFERENCE": - self.layer.append(TF_Parser.parse_squarddiff(op, self.model)) - elif op_code_str == "RSQRT": - self.layer.append(TF_Parser.parse_rsqrt(op, self.model)) - elif op_code_str == "SLICE": - self.layer.append(TF_Parser.parse_slice(op, self.model)) + elif op_code_str in regular_opconverter: + self.layer.append(regular_opconverter[op_code_str](op, self.model)) elif op_code_str in SKIP_OPs: pass else: @@ -188,6 +188,17 @@ def checkIfRequireSEelementmult(self, three_op_sequence): return True return False + # | TRANSPOSE -> MEAN -> MEAN -> | + # | -> AVG_POOL_2D | + # | Fuse Target | + def checIfMergeTwoMean1d(self, two_op_sequence): + if ( + getOpCodeStr(two_op_sequence[0], self.model) == "MEAN" + and getOpCodeStr(two_op_sequence[1], self.model) == "MEAN" + ): + return True + return False + def _convert_PAD(self, op): # get input, weight, and output tensors input_tensors = get_input_tensors(op, self.model) diff --git a/code_generator/converters/tflite_parser/__init__.py b/code_generator/converters/tflite_parser/__init__.py index 1c915f17..67a05785 100644 --- a/code_generator/converters/tflite_parser/__init__.py +++ b/code_generator/converters/tflite_parser/__init__.py @@ -7,7 +7,7 @@ from .equal import parse_equal from .fc import parse_fc from .maxpool import parse_maxpool -from .mean1dto2d import parse_mead1dto2d +from .mean1dto2d import parse_mean1dto2d from .mul import parse_mul from .notequal import parse_notequal from .reshape import parse_reshape_fuse_tensor_tuple diff --git a/code_generator/converters/tflite_parser/mean1dto2d.py b/code_generator/converters/tflite_parser/mean1dto2d.py index a4ece246..1bc9bfcd 100644 --- a/code_generator/converters/tflite_parser/mean1dto2d.py +++ b/code_generator/converters/tflite_parser/mean1dto2d.py @@ -4,7 +4,7 @@ from .utils import get_hwc_from_chwshape, get_input_tensors, get_output_tensors, getTensorTypeStr -def parse_mead1dto2d(op, model: Model.Model, MEAN2Dholder=None): +def parse_mean1dto2d(op, model: Model.Model, MEAN2Dholder=None): # Incase no params input_type = None From c73dcce9e9b626bb7f46564495a28b307f7aba62 Mon Sep 17 00:00:00 2001 From: meenchen Date: Tue, 28 Feb 2023 16:40:58 -0500 Subject: [PATCH 19/19] support ops --- code_generator/TfliteConvertor.py | 46 +++++++--- .../converters/tflite_parser/__init__.py | 3 + .../converters/tflite_parser/concat.py | 32 ++----- .../converters/tflite_parser/equal.py | 9 +- code_generator/converters/tflite_parser/fc.py | 4 +- .../converters/tflite_parser/mean1d.py | 78 +++++++++++++++++ .../converters/tflite_parser/sub.py | 2 +- .../converters/tflite_parser/transpose.py | 73 ++++++++++++++++ .../converters/tflite_parser/utils.py | 6 +- code_generator/operators/batchmatmul.py | 17 ++-- code_generator/operators/concat.py | 66 +++++++++++---- code_generator/operators/div.py | 5 +- code_generator/operators/equal.py | 24 +++++- code_generator/operators/mean1d.py | 83 +++++++++++++++++++ code_generator/operators/notequal.py | 13 ++- code_generator/operators/sub.py | 1 + code_generator/operators/transpose.py | 8 +- 17 files changed, 400 insertions(+), 70 deletions(-) create mode 100644 code_generator/converters/tflite_parser/mean1d.py create mode 100644 code_generator/converters/tflite_parser/transpose.py create mode 100644 code_generator/operators/mean1d.py diff --git a/code_generator/TfliteConvertor.py b/code_generator/TfliteConvertor.py index 9c779289..c3da9f29 100644 --- a/code_generator/TfliteConvertor.py +++ b/code_generator/TfliteConvertor.py @@ -37,7 +37,7 @@ "BATCH_MATMUL": TF_Parser.parse_batchmatmul, "NOT_EQUAL": TF_Parser.parse_notequal, "EQUAL": TF_Parser.parse_equal, - "CONCATENATION": TF_Parser.parse_notequal, + "CONCATENATION": TF_Parser.parse_concat, "CAST": TF_Parser.parse_cast, "SUB": TF_Parser.parse_sub, "MUL": TF_Parser.parse_mul, @@ -45,6 +45,8 @@ "SQUARED_DIFFERENCE": TF_Parser.parse_squarddiff, "RSQRT": TF_Parser.parse_rsqrt, "SLICE": TF_Parser.parse_slice, + "MEAN": TF_Parser.parse_mean1d, + "TRANSPOSE": TF_Parser.parse_transpose, } @@ -127,11 +129,28 @@ def parseOperatorInfo(self): self.layer.append(SEelementmult_op) continue + if i + 2 < operators_len - 2: + next_op = self.subgraph.Operators(i + 1) + next_next_op = self.subgraph.Operators(i + 2) + three_op_sequence = [op, next_op, next_next_op] + + if self.checkIfMergeTransposeTwoMean1d(three_op_sequence): + logging.info("found target to merge transpose and two mean1d") + self._convert_TRANSPOSE(op) + skip_next_ops = 2 + ret_op = TF_Parser.parse_mean1dto2d(next_op, self.model, self.average_1D_to_2D_holder) + ret_op = TF_Parser.parse_mean1dto2d(next_next_op, self.model, self.average_1D_to_2D_holder) + if ret_op is not None: + if self.skip_transpose is not None: + ret_op.params["input_idx"] = self.skip_transpose.input_idx + ret_op.input_tensors[0].graph_idx = self.skip_transpose.input_idx + self.layer.append(ret_op) + continue if i + 1 < operators_len - 1: next_op = self.subgraph.Operators(i + 1) two_op_sequence = [op, next_op] - if self.checIfMergeTwoMean1d(two_op_sequence): + if self.checkIfMergeTwoMean1d(two_op_sequence): logging.info("found target to merge two mean1d") skip_next_ops = 1 ret_op = TF_Parser.parse_mean1dto2d(op, self.model, self.average_1D_to_2D_holder) @@ -141,7 +160,6 @@ def parseOperatorInfo(self): ret_op.params["input_idx"] = self.skip_transpose.input_idx ret_op.input_tensors[0].graph_idx = self.skip_transpose.input_idx self.layer.append(ret_op) - continue # parse the op @@ -161,12 +179,8 @@ def _handleOperator(self, op): self.tmpPADIndice = None elif op_code_str == "PAD": self._convert_PAD(op) - elif op_code_str in "MEAN": - pass - elif op_code_str == "TRANSPOSE": - self._convert_TRANSPOSE(op) - elif op_code_str == "FULLY_CONNECTED": - self.layer.append(TF_Parser.parse_fc(op, self.model)) + # elif op_code_str == "TRANSPOSE": + # self._convert_TRANSPOSE(op) elif op_code_str == "RESHAPE": self.inplace_reshape_table.append(TF_Parser.parse_reshape_fuse_tensor_tuple(op, self.model)) elif op_code_str in regular_opconverter: @@ -191,7 +205,7 @@ def checkIfRequireSEelementmult(self, three_op_sequence): # | TRANSPOSE -> MEAN -> MEAN -> | # | -> AVG_POOL_2D | # | Fuse Target | - def checIfMergeTwoMean1d(self, two_op_sequence): + def checkIfMergeTwoMean1d(self, two_op_sequence): if ( getOpCodeStr(two_op_sequence[0], self.model) == "MEAN" and getOpCodeStr(two_op_sequence[1], self.model) == "MEAN" @@ -199,6 +213,18 @@ def checIfMergeTwoMean1d(self, two_op_sequence): return True return False + # | MEAN -> MEAN -> | + # | -> AVG_POOL_2D | + # | Fuse Target | + def checkIfMergeTransposeTwoMean1d(self, three_op_sequence): + if ( + getOpCodeStr(three_op_sequence[1], self.model) == "TRANSPOSE" + and getOpCodeStr(three_op_sequence[1], self.model) == "MEAN" + and getOpCodeStr(three_op_sequence[2], self.model) == "MEAN" + ): + return True + return False + def _convert_PAD(self, op): # get input, weight, and output tensors input_tensors = get_input_tensors(op, self.model) diff --git a/code_generator/converters/tflite_parser/__init__.py b/code_generator/converters/tflite_parser/__init__.py index 67a05785..a6791559 100644 --- a/code_generator/converters/tflite_parser/__init__.py +++ b/code_generator/converters/tflite_parser/__init__.py @@ -2,11 +2,13 @@ from .avgpool import parse_avgpool from .batchmatmul import parse_batchmatmul from .cast import parse_cast +from .concat import parse_concat from .conv2d import parse_conv2d from .div import parse_div from .equal import parse_equal from .fc import parse_fc from .maxpool import parse_maxpool +from .mean1d import parse_mean1d from .mean1dto2d import parse_mean1dto2d from .mul import parse_mul from .notequal import parse_notequal @@ -17,4 +19,5 @@ from .softmax import parse_softmax from .squarddiff import parse_squarddiff from .sub import parse_sub +from .transpose import parse_transpose from .upsample import parse_upsample diff --git a/code_generator/converters/tflite_parser/concat.py b/code_generator/converters/tflite_parser/concat.py index 64f45ddc..d23c83ab 100644 --- a/code_generator/converters/tflite_parser/concat.py +++ b/code_generator/converters/tflite_parser/concat.py @@ -32,17 +32,13 @@ def parse_concat(op, model: Model.Model): _, input2_h, input2_w, input2_c = get_nhwc_from_shape(input2_tensor.tensor.ShapeAsNumpy()) _, output_h, output_w, output_c = get_nhwc_from_shape(output_tensor.tensor.ShapeAsNumpy()) - assert input_h == output_h, "tensor shape not consistent" - assert input_w == output_w, "tensor shape not consistent" - assert input_c == output_c, "tensor shape not consistent" - # tensor types input_type = getTensorTypeStr(input_tensor.tensor.Type()) input_type2 = getTensorTypeStr(input2_tensor.tensor.Type()) output_type = getTensorTypeStr(output_tensor.tensor.Type()) assert input_type == input_type2 == output_type, "tensor type not consistent" - # Check if the divisor is constant, e.g., a scale value or tensor + # Check if any constant input2_idx = input2_tensor.tensor_idx try: input2 = get_np_from_wrapper(input2_tensor) @@ -50,23 +46,20 @@ def parse_concat(op, model: Model.Model): except Exception: input2 = None - # initialize quantized parameters as None for floating-pointer ops - input_zero_point = None - input_scale = None - input2_zero_point = None - input2_scale = None - output_zero_point = None - output_scale = None - - if "float32" not in [output_type, input_type, input_type2]: - raise NotImplementedError("only support floating point for now.") + input_idx = input_tensor.tensor_idx + try: + input = get_np_from_wrapper(input_tensor) + input_idx = "constant" + str(input_idx) + except Exception: + input = None # assign params params = { # operator "op": op_code_str, # tensor - "input_idx": input_tensor.tensor_idx, + "input": input, + "input_idx": input_idx, "input_size": input_h * input_w * input_c, "input2_idx": input2_idx, "input2": input2, @@ -87,13 +80,6 @@ def parse_concat(op, model: Model.Model): "input_dtype": input_type, "input2_dtype": input_type2, "output_dtype": output_type, - # trainable parameters - "input_zero_point": input_zero_point, - "input2_zero_point": input2_zero_point, - "output_zero_point": output_zero_point, - "input_scale": input_scale, - "input2_scale": input2_scale, - "output_scale": output_scale, } op = concat.concat(params) diff --git a/code_generator/converters/tflite_parser/equal.py b/code_generator/converters/tflite_parser/equal.py index 9aabb53a..8ef8f087 100644 --- a/code_generator/converters/tflite_parser/equal.py +++ b/code_generator/converters/tflite_parser/equal.py @@ -22,13 +22,19 @@ def parse_equal(op, model: Model.Model): _, input2_h, input2_w, input2_c = get_nhwc_from_shape(input_tensors[1].tensor.ShapeAsNumpy()) _, output_h, output_w, output_c = get_nhwc_from_shape(output_tensor.tensor.ShapeAsNumpy()) - # Check if the divisor is constant, e.g., a scale value or tensor + # Check any constant, e.g., a scale value or tensor input2_idx = input_tensors[1].tensor_idx try: input2 = get_np_from_wrapper(input_tensors[1]) input2_idx = "constant" + str(input2_idx) except Exception: input2 = None + input_idx = input_tensors[0].tensor_idx + try: + input = get_np_from_wrapper(input_tensors[0]) + input_idx = "constant" + str(input_idx) + except Exception: + input = None params = { # op related @@ -37,6 +43,7 @@ def parse_equal(op, model: Model.Model): "input2_idx": input2_idx, "output_idx": output_tensors[0].tensor_idx, # tensor related + "input": input, "input_h": input_h, "input_w": input_w, "input_c": input_c, diff --git a/code_generator/converters/tflite_parser/fc.py b/code_generator/converters/tflite_parser/fc.py index 3aa9281b..c543d3ef 100644 --- a/code_generator/converters/tflite_parser/fc.py +++ b/code_generator/converters/tflite_parser/fc.py @@ -32,7 +32,7 @@ def parse_fc(op, model: Model.Model): # shapes _, input_h, input_w, input_c = get_nhwc_from_shape(input_tensor.tensor.ShapeAsNumpy()) _, _, output_c, input_c_dual = get_nhwc_from_shape(weight_tensor.tensor.ShapeAsNumpy()) - _, _, output_h, output_c_dual = get_nhwc_from_shape(output_tensor.tensor.ShapeAsNumpy()) + _, output_h, output_w, output_c_dual = get_nhwc_from_shape(output_tensor.tensor.ShapeAsNumpy()) assert input_c_dual == input_c, "channels not match" assert output_c_dual == output_c, "channels not match" @@ -84,7 +84,7 @@ def parse_fc(op, model: Model.Model): "input_dim": 3, "output_dim": 2, "output_h": output_h, - "output_w": 1, + "output_w": output_w, "output_c": output_c, "input_dtype": input_type, "output_dtype": output_type, diff --git a/code_generator/converters/tflite_parser/mean1d.py b/code_generator/converters/tflite_parser/mean1d.py new file mode 100644 index 00000000..bd6a46db --- /dev/null +++ b/code_generator/converters/tflite_parser/mean1d.py @@ -0,0 +1,78 @@ +from code_generator.operators import mean1d +from code_generator.tflite import Model + +from .utils import get_hwc_from_chwshape, get_input_tensors, get_output_tensors, getTensorTypeStr + + +def parse_mean1d(op, model: Model.Model): + # get input, weight, and output tensors + input_tensors = get_input_tensors(op, model) + input_tensor = input_tensors[0] + + output_tensors = get_output_tensors(op, model) + assert len(output_tensors) == 1, "output tensors length should be 1" + output_tensor = output_tensors[0] + + # shapes + input_shape = input_tensor.tensor.ShapeAsNumpy() + output_shape = output_tensor.tensor.ShapeAsNumpy() + + input_h, input_w, input_c = get_hwc_from_chwshape(input_shape) + output_h, output_w, output_c = get_hwc_from_chwshape(output_shape) + input_type = getTensorTypeStr(input_tensor.tensor.Type()) + output_type = getTensorTypeStr(output_tensor.tensor.Type()) + assert output_type == input_type + + filter_h = input_h - output_h + 1 + filter_w = input_w - output_w + 1 + params = { + # operator + "op": "MEAN_1D", + # pool parameters + "filter_h": filter_h, + "filter_w": filter_w, + "stride_h": 1, + "stride_w": 1, + "pad_h": 0, + "pad_w": 0, + # tensor related + "input_idx": input_tensor.tensor_idx, + "output_idx": output_tensor.tensor_idx, + "input_h": input_h, + "input_w": input_w, + "input_c": input_c, + "output_h": output_h, + "output_w": output_w, + "output_c": output_c, + "input_dtype": input_type, + "output_dtype": output_type, + } + + op = mean1d.mean1d(params) + + return op + + +class MEAN2D(object): + def __init__(self): + self.reset_holder() + + def add_first_1D_op(self, input_idx, output_idx, input_h, input_w, input_c): + self.first_1D_input_idx = input_idx + self.first_1D_output_idx = output_idx + self.input_h = input_h + self.input_w = input_w + self.input_c = input_c + self.has_first_1D = True + + def add_second_1D_op(self, input_idx, output_idx, output_h, output_w, output_c): + self.second_1D_input_idx = input_idx + self.second_1D_output_idx = output_idx + self.output_h = output_h + self.output_w = output_w + self.output_c = output_c + self.has_second_1D = True + + def reset_holder(self): + self.has_first_1D = False + self.has_second_1D = False diff --git a/code_generator/converters/tflite_parser/sub.py b/code_generator/converters/tflite_parser/sub.py index 821b31db..a3e0438f 100644 --- a/code_generator/converters/tflite_parser/sub.py +++ b/code_generator/converters/tflite_parser/sub.py @@ -36,7 +36,7 @@ def parse_sub(op, model: Model.Model): output_type = getTensorTypeStr(output_tensor.tensor.Type()) assert input_type == output_type, "tensor type is not consistent" - # Check if the divisor is constant, e.g., a scale value or tensor + # Check if any is constant, e.g., a scale value or tensor input2_idx = input2_tensor.tensor_idx try: input2 = get_np_from_wrapper(input2_tensor) diff --git a/code_generator/converters/tflite_parser/transpose.py b/code_generator/converters/tflite_parser/transpose.py new file mode 100644 index 00000000..c3fac49a --- /dev/null +++ b/code_generator/converters/tflite_parser/transpose.py @@ -0,0 +1,73 @@ +from code_generator.operators import transpose +from code_generator.tflite import Model + +from .utils import ( + get_input_tensors, + get_nhwc_from_shape, + get_np_from_wrapper, + get_output_tensors, + getOpCodeStr, + getTensorTypeStr, +) + + +def parse_transpose(op, model: Model.Model): + # operator + op_code_str = getOpCodeStr(op, model) + + # get input, weight, and output tensors + input_tensors = get_input_tensors(op, model) + input_tensor_count = len(input_tensors) + assert input_tensor_count == 2, "input should be 2 tensors" + + input_tensor = input_tensors[0] + input2_tensor = input_tensors[1] + + output_tensors = get_output_tensors(op, model) + assert len(output_tensors) == 1, "output tensors length should be 1" + output_tensor = output_tensors[0] + + # shapes + _, input_h, input_w, input_c = get_nhwc_from_shape(input_tensor.tensor.ShapeAsNumpy()) + _, output_h, output_w, output_c = get_nhwc_from_shape(output_tensor.tensor.ShapeAsNumpy()) + + # tensor types + input_type = getTensorTypeStr(input_tensor.tensor.Type()) + output_type = getTensorTypeStr(output_tensor.tensor.Type()) + + # Check if any constant + input2_idx = input2_tensor.tensor_idx + try: + input2 = get_np_from_wrapper(input2_tensor) + input2_idx = "constant" + str(input2_idx) + except Exception: + input2 = None + + input_idx = input_tensor.tensor_idx + assert input2[0] == 0 and len(input2) == 4 + + # assign params + params = { + # operator + "op": op_code_str, + # tensor + "input_idx": input_idx, + "input_size": input_h * input_w * input_c, + "input2_idx": input2_idx, + "permute": input2, + "output_idx": output_tensor.tensor_idx, + "input_h": input_h, + "input_w": input_w, + "input_c": input_c, + "input_dim": 3, + "output_dim": 3, + "output_h": output_h, + "output_w": output_w, + "output_c": output_c, + "dtypte": input_type, + "input_dtype": input_type, + "output_dtype": output_type, + } + op = transpose.transpose(params) + + return op diff --git a/code_generator/converters/tflite_parser/utils.py b/code_generator/converters/tflite_parser/utils.py index 3010218b..d1e11461 100644 --- a/code_generator/converters/tflite_parser/utils.py +++ b/code_generator/converters/tflite_parser/utils.py @@ -35,7 +35,9 @@ def _build_str_map(obj): def get_np_from_wrapper(wrapper): - if wrapper.tensor.Type() == TensorType.INT8: + if wrapper.tensor.Type() == TensorType.BOOL: + dtype = np.bool8 + elif wrapper.tensor.Type() == TensorType.INT8: dtype = np.int8 elif wrapper.tensor.Type() == TensorType.INT32: dtype = np.int32 @@ -43,7 +45,7 @@ def get_np_from_wrapper(wrapper): dtype = np.float32 logging.warn("Support of floating-point tensors are experimental.") else: - raise NotImplementedError("Current implementation only supports int8 and int32") + raise NotImplementedError("Current implementation only supports fp32, int8, int32, bool") data = wrapper.buffer.DataAsNumpy() shape = wrapper.tensor.ShapeAsNumpy() if wrapper.tensor.ShapeLength() != 0 else [] diff --git a/code_generator/operators/batchmatmul.py b/code_generator/operators/batchmatmul.py index 6dbaee37..f7a983bc 100644 --- a/code_generator/operators/batchmatmul.py +++ b/code_generator/operators/batchmatmul.py @@ -56,20 +56,27 @@ def __init__(self, params: dict) -> None: def generate_inference_str(self): params = self.params string = "" - input = f"{self._getBufferstrCast(params['input_buf_add'], params['input_buf_add_offset'])}" + input = self._getBufferstrCast(params["input_buf_add"], params["input_buf_add_offset"], params["input_dtype"]) + input = f"{input}" input_col = self.params["input_col"] input_row = self.params["input_row"] - input2 = f"{self._getBufferstrCast(params['input_buf_add'], params['input_buf_add_offset'])}" + input2 = self._getBufferstrCast( + params["input2_buf_add"], params["input2_buf_add_offset"], dtype=params["input2_dtype"] + ) + input2 = f"{input2}" input2_col = self.params["input2_col"] input2_row = self.params["input2_row"] - output = f"{self._getBufferstrCast(params['output_buf_add'], params['output_buf_add_offset'])}" + output = self._getBufferstrCast( + params["output_buf_add"], params["output_buf_add_offset"], dtype=params["output_dtype"] + ) + output = f"{output}" output_col = self.params["output_col"] output_row = self.params["output_row"] if params["input_dtype"] == "float32": string += ( - f"batchmatmul_fp({params['batch_size']},{input},{input_col},{input_row}," - + f"{input2},{input2_col},{input2_row},{output},{output_col},{output_row});\n" + f"batchmatmul_fp({params['batch_size']},{input2},{input2_col},{input2_row}," + f"{input},{input_col},{input_row},{output},{output_col},{output_row});\n" ) else: raise NotImplementedError diff --git a/code_generator/operators/concat.py b/code_generator/operators/concat.py index 7b53ee26..c8073bc8 100644 --- a/code_generator/operators/concat.py +++ b/code_generator/operators/concat.py @@ -1,6 +1,3 @@ -import logging -import warnings - from .basic_utils import basicOperator, deep_copy_dicts, overwrite_dicts __all__ = ["concat"] @@ -12,6 +9,7 @@ "input2_idx": None, "output_idx": None, # tensor related + "input": None, "input_h": None, "input_w": None, "input_c": None, @@ -43,14 +41,16 @@ def __init__(self, params: dict) -> None: self.params["input_w"], self.params["input_c"], ) - if "constant" not in self.params["input2_idx"]: - self._add_input( - self.params["input2_idx"], - self.params["input2_dtype"], - self.params["input2_h"], - self.params["input2_w"], - self.params["input2_c"], - ) + # TODO: Refactor this + if self.input_tensors[0].constant(): + self.input_tensors[0].set_data(self.params["input"], self.params["input_idx"]) + self._add_input( + self.params["input2_idx"], + self.params["input2_dtype"], + self.params["input2_h"], + self.params["input2_w"], + self.params["input2_c"], + ) # TODO: Refactor this if self.input_tensors[1].constant(): self.input_tensors[1].set_data(self.params["input2"], self.params["input2_idx"]) @@ -62,15 +62,49 @@ def __init__(self, params: dict) -> None: self.params["output_c"], ) - if None in default_params: - warnings.warn(f"parameters are not all set for op {self.params['op']}") - def generate_inference_str(self): params = self.params string = "" - if params["input_dtype"] == "float32": - logging.warn("CONCATENATION still needs implementation.") + if self.input_tensors[0].constant(): + input = f"{self.input_tensors[0].graph_idx}" + else: + input = self._getBufferstrCast( + params["input_buf_add"], params["input_buf_add_offset"], params["input_dtype"] + ) + input = f"{input}" + input_h = f"{str(params['input_h'])}" + input_w = f"{str(params['input_w'])}" + input_c = f"{str(params['input_c'])}" + if self.input_tensors[1].constant(): + input2 = f"{self.input_tensors[1].graph_idx}" + else: + input2 = self._getBufferstrCast( + params["input2_buf_add"], params["input2_buf_add_offset"], dtype=params["input2_dtype"] + ) + input2 = f"{input2}" + input2_h = f"{str(params['input2_h'])}" + input2_w = f"{str(params['input2_w'])}" + input2_c = f"{str(params['input2_c'])}" + output = self._getBufferstrCast( + params["output_buf_add"], params["output_buf_add_offset"], dtype=params["output_dtype"] + ) + output = f"{output}" + output_h = f"{str(params['output_h'])}" + output_w = f"{str(params['output_w'])}" + output_c = f"{str(params['output_c'])}" + + if params["input_dtype"] in ["float32", "int32", "bool"]: + if params["input_dtype"] == "float32": + function_name = "concat_fp" + elif params["input_dtype"] == "int32": + function_name = "concat_int32" + elif params["input_dtype"] == "bool": + function_name = "concat_bool" + string += ( + f"{function_name}({input},{input_h},{input_w},{input_c}," + + f"{input2},{input2_h},{input2_w},{input2_c},{output},{output_h},{output_w},{output_c});\n" + ) else: raise NotImplementedError diff --git a/code_generator/operators/div.py b/code_generator/operators/div.py index 1fe17432..4f64fe9c 100644 --- a/code_generator/operators/div.py +++ b/code_generator/operators/div.py @@ -65,6 +65,7 @@ def generate_inference_str(self): string = "" if params["input_dtype"] == "float32": + function_name = "div_fp" if self.params["scale_from_add"] is not None: scale_divisor = f"{self.params['scale_from_add']}" string += ( @@ -80,6 +81,8 @@ def generate_inference_str(self): if iterable(self.params["input2"]): for v in self.params["input2"]: string += f"{str(v)}," + if len(self.params["input2"]) == 1: + function_name = "div_fp_scaler" else: string += f"{str(self.params['input2'])}," string += "};\n" @@ -89,7 +92,7 @@ def generate_inference_str(self): input2_str = f"{self._getBufferstrCast(params['input2_buf_add'], params['input2_buf_add_offset'])}" string += ( - f"div_fp({self.params['input_size']}," + f"{function_name}({self.params['input_size']}," + f"{self._getBufferstrCast(params['input_buf_add'], params['input_buf_add_offset'])}," + f"{input2_str}," + f"{self._getBufferstrCast(params['output_buf_add'], params['output_buf_add_offset'])});\n" diff --git a/code_generator/operators/equal.py b/code_generator/operators/equal.py index 44708022..70035692 100644 --- a/code_generator/operators/equal.py +++ b/code_generator/operators/equal.py @@ -38,6 +38,9 @@ def __init__(self, params: dict) -> None: self.params["input_w"], self.params["input_c"], ) + # TODO: Refactor this + if self.input_tensors[0].constant(): + self.input_tensors[0].set_data(self.params["input"], self.params["input_idx"]) self._add_input( self.params["input2_idx"], self.params["input2_dtype"], @@ -45,6 +48,9 @@ def __init__(self, params: dict) -> None: self.params["input2_w"], self.params["input2_c"], ) + # TODO: Refactor this + if self.input_tensors[1].constant(): + self.input_tensors[1].set_data(self.params["input2"], self.params["input2_idx"]) self._add_output( self.params["output_idx"], self.params["output_dtype"], @@ -56,18 +62,30 @@ def __init__(self, params: dict) -> None: def generate_inference_str(self): params = self.params string = "" - input = f"{self._getBufferstrCast(params['input_buf_add'], params['input_buf_add_offset'])}" + if self.input_tensors[0].constant(): + input = f"{self.input_tensors[0].graph_idx}" + else: + input = self._getBufferstrCast( + params["input_buf_add"], params["input_buf_add_offset"], params["input_dtype"] + ) + input = f"{input}" input_h = f"{str(params['input_h'])}" input_w = f"{str(params['input_w'])}" input_c = f"{str(params['input_c'])}" if self.input_tensors[1].constant(): input2 = f"{self.input_tensors[1].graph_idx}" else: - input2 = f"{self._getBufferstrCast(params['input2_buf_add'], params['input2_buf_add_offset'])}" + input2 = self._getBufferstrCast( + params["input2_buf_add"], params["input2_buf_add_offset"], dtype=params["input2_dtype"] + ) + input2 = f"{input2}" input2_h = f"{str(params['input2_h'])}" input2_w = f"{str(params['input2_w'])}" input2_c = f"{str(params['input2_c'])}" - output = f"{self._getBufferstrCast(params['output_buf_add'], params['output_buf_add_offset'])}" + output = self._getBufferstrCast( + params["output_buf_add"], params["output_buf_add_offset"], dtype=params["output_dtype"] + ) + output = f"{output}" output_h = f"{str(params['output_h'])}" output_w = f"{str(params['output_w'])}" output_c = f"{str(params['output_c'])}" diff --git a/code_generator/operators/mean1d.py b/code_generator/operators/mean1d.py new file mode 100644 index 00000000..f2b751d9 --- /dev/null +++ b/code_generator/operators/mean1d.py @@ -0,0 +1,83 @@ +# ---------------------------------------------------------------------- +# Project: TinyEngine +# Title: mean1d.py +# +# Reference papers: +# - MCUNet: Tiny Deep Learning on IoT Device, NeurIPS 2020 +# - MCUNetV2: Memory-Efficient Patch-based Inference for Tiny Deep Learning, NeurIPS 2021 +# - MCUNetV3: On-Device Training Under 256KB Memory, NeurIPS 2022 +# Contact authors: +# - Wei-Ming Chen, wmchen@mit.edu +# - Wei-Chen Wang, wweichen@mit.edu +# - Ji Lin, jilin@mit.edu +# - Ligeng Zhu, ligeng@mit.edu +# - Song Han, songhan@mit.edu +# +# Target ISA: ARMv7E-M +# ---------------------------------------------------------------------- + +from .basic_utils import basicOperator, deep_copy_dicts, overwrite_dicts + +__all__ = ["mean1d"] + +default_params = { + # op related + "op": "MEAN_1D", + "filter_h": None, + "filter_w": None, + "stride_h": None, + "stride_w": None, + "pad_h": None, + "pad_w": None, + "input_idx": None, + "output_idx": None, + # tensor related + "input_dim": None, + "input_h": None, + "input_w": None, + "input_c": None, + "output_dim": None, + "output_h": None, + "output_w": None, + "output_c": None, + "input_dtype": None, + "output_dtype": None, +} + + +class mean1d(basicOperator): + def __init__(self, params: dict) -> None: + self.params = deep_copy_dicts(default_params) + overwrite_dicts(self.params, params) + super().__init__() + # handle input/output tensors in HWC format + self._add_input( + self.params["input_idx"], + self.params["input_dtype"], + self.params["input_c"], + self.params["input_w"], + self.params["input_h"], + ) + self._add_output( + self.params["output_idx"], + self.params["output_dtype"], + self.params["output_c"], + self.params["output_w"], + self.params["output_h"], + ) + + def generate_inference_str(self): + params = self.params + if params["input_dtype"] == "float32": + function_name = "mean1d_fp" + else: + raise NotImplementedError + + string = ( + f"{function_name}({self._getBufferstr(params['input_buf_add'], params['input_buf_add_offset'])}," + + f"{str(params['input_h'])},{str(params['input_w'])},{str(params['input_c'])}," + f"{self._getBufferstr(params['output_buf_add'], params['output_buf_add_offset'])}" + + f"{str(params['output_h'])},{str(params['output_w'])},{str(params['output_c'])});\n" + ) + + return string diff --git a/code_generator/operators/notequal.py b/code_generator/operators/notequal.py index 70ff285b..439c13b8 100644 --- a/code_generator/operators/notequal.py +++ b/code_generator/operators/notequal.py @@ -59,18 +59,25 @@ def __init__(self, params: dict) -> None: def generate_inference_str(self): params = self.params string = "" - input = f"{self._getBufferstrCast(params['input_buf_add'], params['input_buf_add_offset'])}" + input = self._getBufferstrCast(params["input_buf_add"], params["input_buf_add_offset"], params["input_dtype"]) + input = f"{input}" input_h = f"{str(params['input_h'])}" input_w = f"{str(params['input_w'])}" input_c = f"{str(params['input_c'])}" if self.input_tensors[1].constant(): input2 = f"{self.input_tensors[1].graph_idx}" else: - input2 = f"{self._getBufferstrCast(params['input2_buf_add'], params['input2_buf_add_offset'])}" + input2 = self._getBufferstrCast( + params["input2_buf_add"], params["input2_buf_add_offset"], dtype=params["input2_dtype"] + ) + input2 = f"{input2}" input2_h = f"{str(params['input2_h'])}" input2_w = f"{str(params['input2_w'])}" input2_c = f"{str(params['input2_c'])}" - output = f"{self._getBufferstrCast(params['output_buf_add'], params['output_buf_add_offset'])}" + output = self._getBufferstrCast( + params["output_buf_add"], params["output_buf_add_offset"], dtype=params["output_dtype"] + ) + output = f"{output}" output_h = f"{str(params['output_h'])}" output_w = f"{str(params['output_w'])}" output_c = f"{str(params['output_c'])}" diff --git a/code_generator/operators/sub.py b/code_generator/operators/sub.py index 3381cfb7..58bf3862 100644 --- a/code_generator/operators/sub.py +++ b/code_generator/operators/sub.py @@ -64,6 +64,7 @@ def generate_inference_str(self): ) else: raise NotImplementedError("sub with scaler/vector constant support is still no ready.") + return string if params["input_dtype"] == "float32": string = ( f"sub({self.params['input_size']}," diff --git a/code_generator/operators/transpose.py b/code_generator/operators/transpose.py index a4ca5db6..acf8b540 100644 --- a/code_generator/operators/transpose.py +++ b/code_generator/operators/transpose.py @@ -9,6 +9,7 @@ "output_idx": None, # tensor related "input_dim": None, + "permute": None, "input_h": None, "input_w": None, "input_c": None, @@ -28,7 +29,7 @@ def __init__(self, params: dict) -> None: overwrite_dicts(self.params, params) super().__init__() # handle input/output tensors in HWC format - if "weight" in self.params["input_idx"]: + if isinstance(self.params["input_idx"], str) and "weight" in self.params["input_idx"]: self.params["input_buf_add"] = self.params["input_idx"] self.params["input_buf_add_offset"] = 0 else: @@ -54,9 +55,10 @@ def generate_inference_str(self): params = self.params if params["input_dtype"] == "float32": string = ( - f"transpose_3Dto3D({self._getBufferstrCast(params['input_buf_add'], params['input_buf_add_offset'])}," + f"transpose4D_fp({self._getBufferstrCast(params['input_buf_add'], params['input_buf_add_offset'])}," ) - string += f"{str(params['input_h'])},{str(params['input_w'])},{str(params['input_c'])}," + string += f"{str(params['permute'][0])},{str(params['permute'][1])},{str(params['permute'][2])}," + string += f"{str(params['permute'][3])}," string += f"{self._getBufferstrCast(params['output_buf_add'], params['output_buf_add_offset'])});\n" else: raise NotImplementedError