[#175] Ensure correct Tokenizer address mode is used consistently, fi…

…x address and response parsing accordingly

clients/java-client-lib/client-command-api/src/main/java/net/zscript/javaclient/commandbuilder/commandnodes/AndSequenceNode.java

@@ -2,12 +2,12 @@
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.stream.Collectors;
 
 import net.zscript.javaclient.commandbuilder.defaultCommands.AbortCommandNode;
 import net.zscript.javaclient.commandbuilder.defaultCommands.BlankCommandNode;
 import net.zscript.javaclient.commandbuilder.defaultCommands.FailureCommandNode;
 import net.zscript.model.components.Zchars;
+import net.zscript.util.ByteString.ByteStringBuilder;
 
 public class AndSequenceNode extends CommandSequenceNode {
     private final List<CommandSequenceNode> elements;
@@ -84,7 +84,8 @@ public List<CommandSequenceNode> getChildren() {
         return elements;
     }
 
-    public String asString() {
-        return elements.stream().map(CommandSequenceNode::asString).collect(Collectors.joining("" + (char) Zchars.Z_ANDTHEN));
+    @Override
+    public void appendTo(ByteStringBuilder builder) {
+        builder.appendJoining(elements, b -> b.appendByte(Zchars.Z_ANDTHEN));
     }
 }

clients/java-client-lib/client-command-api/src/main/java/net/zscript/javaclient/commandbuilder/commandnodes/CommandSequenceNode.java

@@ -8,11 +8,12 @@
 import net.zscript.javaclient.commandbuilder.defaultCommands.AbortCommandNode;
 import net.zscript.javaclient.commandbuilder.defaultCommands.BlankCommandNode;
 import net.zscript.javaclient.commandbuilder.defaultCommands.FailureCommandNode;
+import net.zscript.util.ByteString.ByteAppendable;
 
 /**
  * An element of a Command Sequence under construction, representing a node in the Syntax Tree of a sequence during building.
  */
-public abstract class CommandSequenceNode implements Iterable<ZscriptCommandNode<?>> {
+public abstract class CommandSequenceNode implements Iterable<ZscriptCommandNode<?>>, ByteAppendable {
     private CommandSequenceNode parent = null;
 
     void setParent(CommandSequenceNode parent) {
@@ -149,7 +150,7 @@ public ZscriptCommandNode<?> next() {
                     if (!children.hasNext()) {
                         throw new NoSuchElementException();
                     }
-                    CommandSequenceNode node = children.next();
+                    final CommandSequenceNode node = children.next();
                     if (node instanceof ZscriptCommandNode) {
                         return (ZscriptCommandNode<?>) node;
                     }
@@ -159,6 +160,4 @@ public ZscriptCommandNode<?> next() {
             }
         };
     }
-
-    public abstract String asString();
 }

clients/java-client-lib/client-command-api/src/main/java/net/zscript/javaclient/commandbuilder/commandnodes/OrSequenceNode.java

@@ -2,6 +2,10 @@
 
 import java.util.List;
 
+import net.zscript.model.components.Zchars;
+import net.zscript.util.ByteString;
+import net.zscript.util.ByteString.ByteAppendable;
+
 public class OrSequenceNode extends CommandSequenceNode {
     final CommandSequenceNode before;
     final CommandSequenceNode after;
@@ -22,7 +26,7 @@ protected boolean canFail() {
     boolean isCommand() {
         return false;
     }
- 
+
     @Override
     CommandSequenceNode optimize() {
         if (!before.canFail()) {
@@ -35,7 +39,12 @@ public List<CommandSequenceNode> getChildren() {
         return List.of(before, after);
     }
 
-    public String asString() {
-        return "(" + before.asString() + "|" + after.asString() + ")";
+    @Override
+    public void appendTo(ByteString.ByteStringBuilder builder) {
+        builder.appendByte(Zchars.Z_OPENPAREN)
+                .append((ByteAppendable) before)
+                .appendByte(Zchars.Z_ORELSE)
+                .append((ByteAppendable) after)
+                .appendByte(Zchars.Z_CLOSEPAREN);
     }
 }

clients/java-client-lib/client-command-api/src/main/java/net/zscript/javaclient/commandbuilder/commandnodes/ZscriptCommandNode.java

@@ -5,16 +5,16 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 
+import static java.util.stream.Collectors.toList;
 import static net.zscript.javaclient.commandPaths.NumberField.fieldOf;
 
 import net.zscript.javaclient.commandPaths.BigField;
 import net.zscript.javaclient.commandPaths.ZscriptFieldSet;
 import net.zscript.javaclient.commandbuilder.Respondable;
 import net.zscript.javaclient.commandbuilder.ZscriptResponse;
 import net.zscript.tokenizer.ZscriptExpression;
-import net.zscript.util.ByteString;
+import net.zscript.util.ByteString.ByteStringBuilder;
 
 public abstract class ZscriptCommandNode<T extends ZscriptResponse> extends CommandSequenceNode implements Respondable<T> {
 
@@ -58,14 +58,12 @@ public List<CommandSequenceNode> getChildren() {
 
     @Nonnull
     public ZscriptFieldSet asFieldSet() {
-        return ZscriptFieldSet.fromMap(bigFields.stream().map(BigField::getData).collect(Collectors.toList()),
-                bigFields.stream().map(BigField::isString).collect(Collectors.toList()), fields);
+        return ZscriptFieldSet.fromMap(bigFields.stream().map(BigField::getData).collect(toList()),
+                bigFields.stream().map(BigField::isString).collect(toList()), fields);
     }
 
-    @Nonnull
     @Override
-    public String asString() {
-        ByteString.ByteStringBuilder b = ByteString.builder();
+    public void appendTo(ByteStringBuilder b) {
         for (int i = 'A'; i <= 'Z'; i++) {
             if (fields.get((byte) i) != null) {
                 int value = fields.get((byte) i);
@@ -75,6 +73,5 @@ public String asString() {
         for (BigField f : bigFields) {
             f.appendTo(b);
         }
-        return b.asString();
     }
 }

clients/java-client-lib/client-command-api/src/test/java/net/zscript/javaclient/commandbuilder/commandNodes/SequenceNodeTest.java

@@ -1,13 +1,10 @@
 package net.zscript.javaclient.commandbuilder.commandNodes;
 
-import java.nio.charset.StandardCharsets;
-
 import static org.assertj.core.api.Assertions.assertThat;
 
 import org.junit.jupiter.api.Test;
 
 import net.zscript.javaclient.commandbuilder.DemoActivateCommand;
-import net.zscript.javaclient.commandbuilder.commandnodes.CommandSequenceNode;
 
 public class SequenceNodeTest {
     @Test
@@ -16,7 +13,7 @@ public void shouldCompileBasicCommand() {
                 .setVersionType(DemoCapabilitiesCommandBuilder.PLATFORM_FIRMWARE)
                 .addBigField("abc")
                 .build();
-        assertThat(seq.asString()).isEqualTo("V2Z\"abc\"");
+        assertThat(seq.asStringUtf8()).isEqualTo("V2Z\"abc\"");
     }
 
     @Test
@@ -28,7 +25,7 @@ public void shouldCompileAndedCommandSequence() {
                 .andThen(DemoActivateCommand.builder()
                         .setField('A', 0x1234)
                         .build());
-        assertThat(seq.asString()).isEqualTo("V2Z+0506077f&A1234Z2");
+        assertThat(seq.asStringUtf8()).isEqualTo("V2Z+0506077f&A1234Z2");
     }
 
 }

clients/java-client-lib/client-command-builders/src/test/java/net/zscript/model/modules/base/CoreModuleTest.java

@@ -6,52 +6,51 @@
 
 import org.junit.jupiter.api.Test;
 
-import net.zscript.javaclient.commandbuilder.commandnodes.ZscriptCommandNode;
 import net.zscript.javaclient.commandbuilder.ZscriptMissingFieldException;
+import net.zscript.javaclient.commandbuilder.commandnodes.ZscriptCommandNode;
+import net.zscript.model.modules.base.CoreModule.ActivateCommand.ActivateResponse;
+import net.zscript.model.modules.base.CoreModule.CapabilitiesCommand.CapabilitiesResponse;
+import net.zscript.model.modules.base.CoreModule.EchoCommand.EchoResponse;
+import net.zscript.model.modules.base.CoreModule.ReadIdCommand.ReadIdResponse;
 
 public class CoreModuleTest {
     @Test
     public void shouldCreateCoreCapabilities() {
-        ZscriptCommandNode c = CoreModule.capabilitiesBuilder()
+        ZscriptCommandNode<CapabilitiesResponse> c = CoreModule.capabilitiesBuilder()
                 .build();
-        String ztext = c.asString();
-        assertThat(ztext).isEqualTo("Z");
+        assertThat(c.asStringUtf8()).isEqualTo("Z");
     }
 
     @Test
     public void shouldCreateCoreActivate() {
-        ZscriptCommandNode c = CoreModule.activateBuilder()
+        ZscriptCommandNode<ActivateResponse> c = CoreModule.activateBuilder()
                 .build();
-        String ztext = c.asString();
-        assertThat(ztext).isEqualTo("Z2");
+        assertThat(c.asStringUtf8()).isEqualTo("Z2");
     }
 
     @Test
     public void shouldCreateCoreActivateWithField() {
-        ZscriptCommandNode c = CoreModule.ActivateCommand.builder()
+        ZscriptCommandNode<ActivateResponse> c = CoreModule.ActivateCommand.builder()
                 .setChallenge(3)
                 .build();
-        String ztext = c.asString();
-        assertThat(ztext).isEqualTo("K3Z2");
+        assertThat(c.asStringUtf8()).isEqualTo("K3Z2");
     }
 
     @Test
     public void shouldCreateCoreEcho() {
-        ZscriptCommandNode c = CoreModule.echoBuilder()
+        ZscriptCommandNode<EchoResponse> c = CoreModule.echoBuilder()
                 .setAny('J', 123)
                 .build();
-        String ztext = c.asString();
-        assertThat(ztext).isEqualTo("J7bZ1");
+        assertThat(c.asStringUtf8()).isEqualTo("J7bZ1");
     }
 
     @Test
     public void shouldCreateCoreMatchCodeWithRequiredField() {
-        ZscriptCommandNode c = CoreModule.readIdBuilder()
+        ZscriptCommandNode<ReadIdResponse> c = CoreModule.readIdBuilder()
                 .setIdType(TemporaryId)
                 .setMatchId(new byte[] { 0x3a, 0x42 })
                 .build();
-        String ztext = c.asString();
-        assertThat(ztext).isEqualTo("IZ4+3a42");
+        assertThat(c.asStringUtf8()).isEqualTo("IZ4+3a42");
     }
 
     @Test

clients/java-client-lib/client-command-builders/src/test/java/net/zscript/model/modules/testing/test/JavaCommandBuilderBuildTest.java

@@ -271,7 +271,6 @@ void shouldCreateCommandWithOptionalBytesAsEmptyBytes() {
     }
 
     private String build(ZscriptCommandBuilder<?> b) {
-        // ISO8859, because we want an 8-bit byte in each char, and they *could* be non-printing / non-ascii if they're in Text
-        return b.build().asString();
+        return b.build().asStringUtf8();
     }
 }

clients/java-client-lib/client-command-builders/src/test/java/net/zscript/model/modules/testing/test/JavaCommandBuilderNotificationTest.java

@@ -34,7 +34,7 @@ void shouldDefineNotificationClasses() {
 
     @Test
     void shouldCreateNotificationWithRequiredFields() {
-        final CompleteAddressedResponse car = CompleteAddressedResponse.parse(tokenize(byteStringUtf8("!234 Dab Lcd & Xef\n")).getTokenReader());
+        final CompleteAddressedResponse car = CompleteAddressedResponse.parse(tokenize(byteStringUtf8("!234 Dab Lcd & Xef\n")).getTokenReader().getFirstReadToken());
 
         final TestingModule.TestNtfBNotification.TestNtfBNotificationHandle handle = TestingModule.TestNtfBNotification.ID.newHandle();
 

clients/java-client-lib/client-command-builders/src/test/java/net/zscript/model/modules/testing/test/JavaCommandBuilderResponseTest.java

@@ -25,7 +25,7 @@
 
 public class JavaCommandBuilderResponseTest {
     final ExtendingTokenBuffer buffer      = new ExtendingTokenBuffer();
-    final Tokenizer            tokenizer   = new Tokenizer(buffer.getTokenWriter(), 2);
+    final Tokenizer            tokenizer   = new Tokenizer(buffer.getTokenWriter(), true);
     final TokenReader          tokenReader = buffer.getTokenReader();
 
     @Test

clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/addressing/CompleteAddressedResponse.java

@@ -1,16 +1,13 @@
 package net.zscript.javaclient.addressing;
 
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
 
-import static net.zscript.tokenizer.TokenBuffer.TokenReader;
-import static net.zscript.tokenizer.TokenBuffer.TokenReader.ReadToken;
-
+import net.zscript.javaclient.ZscriptParseException;
 import net.zscript.javaclient.sequence.ResponseSequence;
 import net.zscript.model.components.Zchars;
+import net.zscript.tokenizer.TokenBuffer.TokenReader.ReadToken;
 import net.zscript.tokenizer.Tokenizer;
-import net.zscript.util.OptIterator;
 
 /**
  * Defines a Response sequence along with its address - this is the completed parse from a TokenReader.
@@ -19,30 +16,22 @@ public class CompleteAddressedResponse {
     private final List<ZscriptAddress> addressSections;
     private final ResponseSequence     content;
 
-    public static CompleteAddressedResponse parse(TokenReader reader) {
-        OptIterator<ReadToken> iterEnding = reader.tokenIterator();
-        for (Optional<ReadToken> opt = iterEnding.next(); opt.isPresent(); opt = iterEnding.next()) {
-            if (opt.get().isSequenceEndMarker()) {
-                if (opt.get().getKey() != Tokenizer.NORMAL_SEQUENCE_END) {
-                    throw new RuntimeException("Parse failed with Tokenizer error: " + Integer.toHexString(opt.get().getKey()));
-                }
-            }
-        }
-        OptIterator<ReadToken> iter      = reader.tokenIterator();
-        List<ZscriptAddress>   addresses = new ArrayList<>();
-        ResponseSequence       seq       = null;
-        for (Optional<ReadToken> opt = iter.next(); opt.isPresent(); opt = iter.next()) {
-            ReadToken token = opt.get();
-            if (token.getKey() == Zchars.Z_ADDRESSING) {
-                addresses.add(ZscriptAddress.parse(token));
-            } else {
-                seq = ResponseSequence.parse(token);
-                break;
-            }
-        }
-        if (seq == null) {
-            seq = ResponseSequence.empty();
+    public static CompleteAddressedResponse parse(ReadToken start) {
+        final Optional<ReadToken> endToken = start.tokenIterator().stream()
+                .filter(ReadToken::isSequenceEndMarker).findFirst();
+        if (endToken.isEmpty()) {
+            throw new ZscriptParseException("Parse failed, no terminating sequence marker");
+        } else if (endToken.get().getKey() != Tokenizer.NORMAL_SEQUENCE_END) {
+            throw new ZscriptParseException("Parse failed with Tokenizer error [marker=%s]", endToken.get());
         }
+
+        final List<ZscriptAddress> addresses = ZscriptAddress.parseAll(start);
+        final ResponseSequence seq = start.tokenIterator().stream()
+                .filter(t -> !Zchars.isAddressing(t.getKey()))
+                .findFirst()
+                .map(ResponseSequence::parse)
+                .orElseGet(ResponseSequence::empty);
+
         return new CompleteAddressedResponse(addresses, seq);
     }
 

clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/addressing/ZscriptAddress.java

@@ -1,8 +1,11 @@
 package net.zscript.javaclient.addressing;
 
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 
+import static java.util.stream.Collectors.toList;
 import static net.zscript.javaclient.commandPaths.NumberField.fieldOf;
 import static net.zscript.tokenizer.TokenBuffer.TokenReader.ReadToken;
 
@@ -47,18 +50,31 @@ public static ZscriptAddress from(List<Integer> addressParts) {
         return new ZscriptAddress(addressParts.stream().mapToInt(i -> i).toArray());
     }
 
-    public static ZscriptAddress parse(ReadToken token) {
-        if (token.getKey() != Zchars.Z_ADDRESSING) {
-            throw new IllegalArgumentException("Cannot parse address from non-address fields");
+    public static List<ZscriptAddress> parseAll(ReadToken start) {
+        if (start.getKey() != Zchars.Z_ADDRESSING) {
+            return Collections.emptyList();
         }
 
-        // Note, tokenizer will only write one Address; subsequent addresses are inside the single token envelope.
-        int[] parts = token.tokenIterator().stream()
-                .takeWhile(t -> Zchars.isAddressing(t.getKey()))
-                .mapToInt(ReadToken::getData16)
-                .toArray();
+        final List<List<Integer>> addresses = new ArrayList<>();
+        List<Integer>             elements  = null;
 
-        return new ZscriptAddress(parts);
+        for (ReadToken token : start.tokenIterator().toIterable()) {
+            if (token.getKey() == Zchars.Z_ADDRESSING) {
+                elements = new ArrayList<>();
+                addresses.add(elements);
+                elements.add(token.getData16());
+            } else if (token.getKey() == Zchars.Z_ADDRESSING_CONTINUE) {
+                elements.add(token.getData16());
+            } else {
+                break;
+            }
+        }
+
+        return addresses.stream()
+                .map(list -> new ZscriptAddress(list.stream()
+                        .mapToInt(i -> i)
+                        .toArray()))
+                .collect(toList());
     }
 
     private ZscriptAddress(int[] addr) {
@@ -108,15 +124,6 @@ public final boolean equals(Object obj) {
         return Arrays.equals(addressParts, other.addressParts);
     }
 
-    /**
-     * Presents this address in conventional "@3.6a.1" format, including empty field for zero.
-     *
-     * @return the address as a string
-     */
-    public String asString() {
-        return toByteString().asString();
-    }
-
     @Override
     public String toString() {
         return toStringImpl();