Another round of accumulated tweaks and cleanups

Notable changes:

src/cmd/ksh93/*.c:
- Get rid of all the dtuserdata(FOO,&sh,1) calls backported in
  cc49275. These set pointers to sh in Cdt objects. As of 4491bc6
  and its follow-up commits, the code does not use any pointers to
  sh, so these are superfluous.

src/cmd/ksh93/data/builtins.c,
src/cmd/ksh93/sh.1:
- As of ksh 93l 2001-06-01, the -h/trackall option has no effect at
  all, so trim its documentation.

src/lib/libast/man/stk.3,
src/lib/libast/man/stak.3:
- Correct the documentation on what the ST(A)K_SMALL option bit
  actually does based on a reading of the code.
- Document the STK_NULL option bit.

README.md,
src/cmd/ksh93/README:
- Add a note that -fdiagnostics-color=always will break the build.
  Ref.: ksh93/ksh#379

src/lib/libast/Mamfile:
- Remove a 'rm -f astmath' command -- a file that is never created.
  But on Cygwin this removes astmath.exe, which *is* used. As a
  result, executing it failed on Cygwin, so the system incorrectly
  detected that Cygwin needs -lm for math functions.

COPYRIGHT

@@ -3,7 +3,7 @@ ksh 93u+m general copyright notice
 ########################################################################
 #                                                                      #
 #                   The KornShell 93u+m distribution                   #
-#             Copyright (c) 2021 Contributors to ksh 93u+m             #
+#          Copyright (c) 2020-2022 Contributors to ksh 93u+m           #
 #                    <https://github.com/ksh93/ksh>                    #
 #             Derived from AT&T's ast package (see below)              #
 #       Licensed under the Eclipse Public License, Version 1.0         #
@@ -50,6 +50,8 @@ ast package general copyright notice
 #                 Glenn Fowler <[email protected]>                  #
 #                  David Korn <[email protected]>                   #
 #                   Phong Vo <[email protected]>                    #
+#                Roland Mainz <[email protected]>               #
+#               Karsten Fleischer <[email protected]>              #
 #                 Adam Edgar <[email protected]>                 #
 #                Adam Buchsbaum <[email protected]>                #
 #                Aman Shaikh <[email protected]>                #

README.md

@@ -42,7 +42,7 @@ Between 2017 and 2020 there was an ultimately unsuccessful
 [attempt](https://github.com/att/ast/tree/2020.0.1)
 to breathe new life into the KornShell by extensively refactoring the last
 unstable AST beta version (93v-).
-While that ksh2020 branch is now abandoned and still has many critical bugs,
+While that ksh2020 effort is now abandoned and still has many critical bugs,
 it also had a lot of bugs fixed. More importantly, the AST issue tracker
 now contains a lot of documentation on how to fix those bugs, which made
 it possible to backport many of them to the last stable release instead.
@@ -57,8 +57,10 @@ as well as many new fixes from the community
 [2](https://github.com/ksh93/ksh/issues?q=is%3Aissue+is%3Aclosed+label%3Abug)).
 Though there are many
 [bugs left to fix](https://github.com/ksh93/ksh/issues),
-we are confident at this point that 93u+m is already the least buggy branch
+we are confident at this point that 93u+m is already the least buggy version
 of ksh93 ever released.
+As of late 2021, distributions such as Debian and Slackware have begun
+to package it as their default version of ksh93.
 
 ## Build
 
@@ -92,6 +94,10 @@ to run the build scripts this way. For example:
 bin/package make SHELL=/bin/bash CCFLAGS="-O2 -I/opt/local/include" LDFLAGS="-L/opt/local/lib"
 ```
 
+**Note:** Do not add compiler flags that cause the compiler to emit terminal
+escape codes, such as `-fdiagnostics-color=always`; this will cause the
+build to fail as the probing code greps compiler diagnostics.
+
 For more information run
 ```sh
 bin/package help

src/cmd/ksh93/README

@@ -192,6 +192,10 @@ run the build scripts this way. For example:
 	bin/package make SHELL=/bin/bash CCFLAGS="-O2 -I/opt/local/include" \
 		LDFLAGS="-L/opt/local/lib"
 
+Note: Do not add compiler flags that cause the compiler to emit terminal
+escape codes, such as -fdiagnostics-color=always; this will cause the build
+to fail as the probing code greps compiler diagnostics.
+
 For more information, run:
 
 	bin/package help

src/cmd/ksh93/data/builtins.c

@@ -180,9 +180,8 @@ const char sh_set[] =
 	"[++?contained in the pipeline following \b!\b.]"
 "}"
 "[f?Pathname expansion is disabled.]"
-"[h?Obsolete.  Causes each command whose name has the syntax of an "
-	"alias to become a tracked alias when it is first encountered.]"
-"[k?This is obsolete.  All arguments of the form \aname\a\b=\b\avalue\a "
+"[h?Obsolete; no effect.]"
+"[k?All arguments of the form \aname\a\b=\b\avalue\a "
 	"are removed and placed in the variable assignment list for "
 	"the command.  Ordinarily, variable assignments must precede "
 	"command arguments.]"

src/cmd/ksh93/sh.1

@@ -7296,11 +7296,9 @@ This mode is disabled while reading profiles.
 Disables pathname expansion.
 .TP 8
 .B \-h
-Each command
-becomes a tracked alias when first encountered.
+Obsolete; no effect.
 .TP 8
 .B \-k
-(Obsolete).
 All variable assignment arguments are placed in the environment for a command,
 not just those that precede the command name.
 .TP 8

src/cmd/ksh93/sh/arith.c

@@ -180,10 +180,7 @@ static Namval_t *scope(register Namval_t *np,register struct lval *lvalue,int as
 			{
 				ap = nv_arrayptr(np);
 				if(ap && !ap->table)
-				{
 					ap->table = dtopen(&_Nvdisc,Dtoset);
-					dtuserdata(ap->table,&sh,1);
-				}
 				if(ap && ap->table && (nq=nv_search(nv_getsub(np),ap->table,NV_ADD)))
 					nq->nvenv = (char*)np;
 				if(nq && nv_isnull(nq))

src/cmd/ksh93/sh/array.c

@@ -94,7 +94,6 @@ static Namarr_t *array_scope(Namval_t *np, Namarr_t *ap, int flags)
 	if(is_associative(aq))
 	{
 		aq->scope = (void*)dtopen(&_Nvdisc,Dtoset);
-		dtuserdata(aq->scope,&sh,1);
 		dtview((Dt_t*)aq->scope,aq->table);
 		aq->table = (Dt_t*)aq->scope;
 		return(aq);
@@ -379,10 +378,7 @@ static Namval_t *array_find(Namval_t *np,Namarr_t *arp, int flag)
 		{
 			char *cp;
 			if(!ap->header.table)
-			{
 				ap->header.table = dtopen(&_Nvdisc,Dtoset);
-				dtuserdata(ap->header.table,&sh,1);
-			}
 			sfprintf(sh.strbuf,"%d",ap->cur);
 			cp = sfstruse(sh.strbuf);
 			mp = nv_search(cp, ap->header.table, NV_ADD);
@@ -415,10 +411,7 @@ int nv_arraysettype(Namval_t *np, Namval_t *tp, const char *sub, int flags)
 	Namarr_t	*ap = nv_arrayptr(np);
 	sh.last_table = 0;
 	if(!ap->table)
-	{
 		ap->table = dtopen(&_Nvdisc,Dtoset);
-		dtuserdata(ap->table,&sh,1);
-	}
 	if(nq = nv_search(sub, ap->table, NV_ADD))
 	{
 		char	*saved_value = NIL(char*);
@@ -481,7 +474,6 @@ static Namfun_t *array_clone(Namval_t *np, Namval_t *mp, int flags, Namfun_t *fp
 	if(ap->table)
 	{
 		ap->table = dtopen(&_Nvdisc,Dtoset);
-		dtuserdata(ap->table,&sh,1);
 		if(ap->scope && !(flags&NV_COMVAR))
 		{
 			ap->scope = ap->table;
@@ -863,7 +855,6 @@ static struct index_array *array_grow(Namval_t *np, register struct index_array
 		if(nv_hasdisc(np,&array_disc) || (nv_type(np) && nv_isvtree(np)))
 		{
 			ap->header.table = dtopen(&_Nvdisc,Dtoset);
-			dtuserdata(ap->header.table,&sh,1);
 			mp = nv_search("0", ap->header.table,NV_ADD);
 			if(mp && nv_isnull(mp))
 			{
@@ -1248,10 +1239,7 @@ Namval_t *nv_putsub(Namval_t *np,register char *sp,register long mode)
 					char *cp;
 					Namval_t *mp;
 					if(!ap->header.table)
-					{
 						ap->header.table = dtopen(&_Nvdisc,Dtoset);
-						dtuserdata(ap->header.table,&sh,1);
-					}
 					sfprintf(sh.strbuf,"%d",ap->cur);
 					cp = sfstruse(sh.strbuf);
 					mp = nv_search(cp, ap->header.table, NV_ADD);
@@ -1676,7 +1664,6 @@ void *nv_associative(register Namval_t *np,const char *sp,int mode)
 	    case NV_AINIT:
 		ap = (struct assoc_array*)sh_calloc(1,sizeof(struct assoc_array));
 		ap->header.table = dtopen(&_Nvdisc,Dtoset);
-		dtuserdata(ap->header.table,&sh,1);
 		ap->cur = 0;
 		ap->pos = 0;
 		ap->header.hdr.disc = &array_disc;

src/cmd/ksh93/sh/init.c

@@ -1805,13 +1805,9 @@ static Init_t *nv_init(void)
 	(OPTINDNOD)->nvalue.lp = (&sh.st.optindex);
 	/* set up the seconds clock */
 	sh.alias_tree = dtopen(&_Nvdisc,Dtoset);
-	dtuserdata(sh.alias_tree,&sh,1);
 	sh.track_tree = dtopen(&_Nvdisc,Dtset);
-	dtuserdata(sh.track_tree,&sh,1);
 	sh.bltin_tree = sh_inittree((const struct shtable2*)shtab_builtins);
-	dtuserdata(sh.bltin_tree,&sh,1);
 	sh.fun_base = sh.fun_tree = dtopen(&_Nvdisc,Dtoset);
-	dtuserdata(sh.fun_tree,&sh,1);
 	dtview(sh.fun_tree,sh.bltin_tree);
 	nv_mount(DOTSHNOD, "type", sh.typedict=dtopen(&_Nvdisc,Dtoset));
 	nv_adddisc(DOTSHNOD, shdiscnames, (Namval_t**)0);
@@ -1853,7 +1849,6 @@ Dt_t *sh_inittree(const struct shtable2 *name_vals)
 		nbltins = n;
 	}
 	base_treep = treep = dtopen(&_Nvdisc,Dtoset);
-	dtuserdata(treep,&sh,1);
 	treep->user = (void*)&sh;
 	for(tp=name_vals;*tp->sh_name;tp++,np++)
 	{

src/cmd/ksh93/sh/macro.c

@@ -2730,10 +2730,7 @@ static char *sh_tilde(register const char *string)
 skip:
 #endif /* _WINIX */
 	if(!logins_tree)
-	{
 		logins_tree = dtopen(&_Nvdisc,Dtbag);
-		dtuserdata(logins_tree,&sh,1);
-	}
 	if(np=nv_search(string,logins_tree,NV_ADD))
 	{
 		save = sh.subshell;

src/cmd/ksh93/sh/main.c

@@ -174,7 +174,6 @@ int sh_main(int ac, char *av[], Shinit_f userinit)
 				/* preset aliases for interactive non-POSIX ksh */
 				dtclose(sh.alias_tree);
 				sh.alias_tree = sh_inittree(shtab_aliases);
-				dtuserdata(sh.alias_tree,&sh,1);
 			}
 		}
 #if SHOPT_REMOTE

src/cmd/ksh93/sh/name.c

@@ -811,7 +811,6 @@ Namval_t *nv_create(const char *name,  Dt_t *root, int flags, Namfun_t *dp)
 				{
 					Dt_t *dp = dtview(sh.var_tree,(Dt_t*)0);
 					rp->sdict = dtopen(&_Nvdisc,Dtoset);
-					dtuserdata(rp->sdict,&sh,1);
 					dtview(rp->sdict,dp);
 					dtview(sh.var_tree,rp->sdict);
 				}
@@ -1164,10 +1163,7 @@ Namval_t *nv_create(const char *name,  Dt_t *root, int flags, Namfun_t *dp)
 								ap = nv_arrayptr(np);
 							}
 							if(n && ap && !ap->table)
-							{
 								ap->table = dtopen(&_Nvdisc,Dtoset);
-								dtuserdata(ap->table,&sh,1);
-							}
 							if(ap && ap->table && (nq=nv_search(sub,ap->table,n)))
 								nq->nvenv = (char*)np;
 							if(nq && nv_isnull(nq))
@@ -2344,7 +2340,6 @@ void sh_scope(struct argnod *envlist, int fun)
 		newroot = nv_dict(sh.namespace);
 #endif /* SHOPT_NAMESPACE */
 	newscope = dtopen(&_Nvdisc,Dtoset);
-	dtuserdata(newscope,&sh,1);
 	if(envlist)
 	{
 		dtview(newscope,(Dt_t*)sh.var_tree);
@@ -3334,10 +3329,7 @@ int nv_rename(register Namval_t *np, int flags)
 		if(ap=nv_arrayptr(np))
 		{
 			if(!ap->table)
-			{
 				ap->table = dtopen(&_Nvdisc,Dtoset);
-				dtuserdata(ap->table,&sh,1);
-			}
 			if(ap->table)
 				mp = nv_search(nv_getsub(np),ap->table,NV_ADD);
 			nv_arraychild(np,mp,0);

src/cmd/ksh93/sh/nvdisc.c

@@ -1326,7 +1326,6 @@ static Namfun_t *clone_table(Namval_t* np, Namval_t *mp, int flags, Namfun_t *fp
 	Dt_t		*oroot=tp->dict,*nroot=dtopen(&_Nvdisc,Dtoset);
 	if(!nroot)
 		return(0);
-	dtuserdata(nroot,dtuserdata(oroot,0,0),1);
 	memcpy((void*)ntp,(void*)fp,sizeof(struct table));
 	ntp->dict = nroot;
 	ntp->parent = nv_lastdict();

src/cmd/ksh93/sh/path.c

@@ -1216,7 +1216,7 @@ pid_t path_spawn(const char *opath,register char **argv, char **envp, Pathcomp_t
 #endif /* SHELLMAGIC */
 #if __CYGWIN__
 	/*
-	 * On Cygwin, execve(2) happily executes shell scripts without a #! path with /bin/sh (which is bash --posix).
+	 * On Cygwin, execve(2) happily executes shell scripts without a #! path with bash (which violates POSIX).
 	 * However, ksh relies on execve(2) executing binaries or #! only, as it uses an ENOEXEC failure to decide
 	 * whether to fork and execute a #!-less shell script with a reinitialized copy of itself via exscript() below.
 	 * So, simulate that failure if the file is not a Windows executable or a script with a #! path.

src/cmd/ksh93/sh/subshell.c

@@ -412,7 +412,6 @@ Dt_t *sh_subtracktree(int create)
 		if(sp && !sp->strack)
 		{
 			sp->strack = dtopen(&_Nvdisc,Dtset);
-			dtuserdata(sp->strack,&sh,1);
 			dtview(sp->strack,sh.track_tree);
 			sh.track_tree = sp->strack;
 		}
@@ -432,7 +431,6 @@ Dt_t *sh_subfuntree(int create)
 		if(sp && !sp->sfun)
 		{
 			sp->sfun = dtopen(&_Nvdisc,Dtoset);
-			dtuserdata(sp->sfun,&sh,1);
 			dtview(sp->sfun,sh.fun_tree);
 			sh.fun_tree = sp->sfun;
 		}

src/cmd/ksh93/sh/xec.c

@@ -2496,7 +2496,6 @@ int sh_exec(register const Shnode_t *t, int flags)
 				else
 				{
 					root = dtopen(&_Nvdisc,Dtoset);
-					dtuserdata(root,&sh,1);
 					nv_mount(np, (char*)0, root);
 					np->nvalue.cp = Empty;
 					dtview(root,sh.var_base);
@@ -2615,10 +2614,7 @@ int sh_exec(register const Shnode_t *t, int flags)
 					if(!sh.fpathdict)
 						sh.fpathdict = dtopen(&_Rpdisc,Dtobag);
 					if(sh.fpathdict)
-					{
-						dtuserdata(sh.fpathdict,&sh,1);
 						dtinsert(sh.fpathdict,rp);
-					}
 				}
 			}
 			else

src/lib/libast/Mamfile

@@ -359,7 +359,6 @@ make install
 					exec - 0) rm -f astmath.exe ;;
 					exec - *) touch astmath.exe ;;
 					exec - esac
-					exec - rm -f astmath
 				done astmath.exe dontcare generated
 				prev FEATURE/aso
 				exec - sed -e '/^#define _REQ_/!d' -e 's/#define _REQ_\([a-z0-9_]*\).*/ -l\1/' FEATURE/aso >> ast.req

src/lib/libast/man/stak.3

@@ -54,9 +54,9 @@ There is a preset initial active stack.
 To use an additional stack, it is necessary to create it and to
 install it as the active stack.
 A stack is created with the \f3stakcreate\fP() function.
-A \fIflags\fP argument of \f3STAK_SMALL\fP indicates that unused
-space on the stack should be freed whenever this stack ceases
-to be the active stack.
+The \fIflags\fP argument is an options bitmask.
+If the \f3STAK_SMALL\fP bit is set, the stack allocates memory in
+small blocks, optimizing for memory usage at the expense of performance.
 If successful,
 \f3stakcreate\fP() returns a pointer to a stack whose reference
 count is 1.

src/lib/libast/man/stk.3

@@ -55,9 +55,11 @@ There is a preset initial active stack.
 To use an additional stack, it is necessary to create it and to
 install it as the active stack.
 A stack is created with the \f3stkopen\fP() function.
-A \fIflags\fP argument of \f3STK_SMALL\fP indicates that unused
-space on the stack should be freed whenever this stack ceases
-to be the active stack.
+The \fIflags\fP argument is an options bitmask.
+If the \f3STK_SMALL\fP bit is set, the stack allocates memory in
+small blocks, optimizing for memory usage at the expense of performance.
+If the \f3STK_NULL\fP bit is set, a stack overflow will cause stack
+operations to return a null pointer instead of throwing an error.
 If successful,
 \f3stkopen\fP() returns a pointer to a stack whose reference
 count is 1.

src/lib/libast/misc/stk.c

@@ -229,11 +229,9 @@ Sfio_t *stkopen(int flags)
 	if(flags&STK_NULL) sp->stkoverflow = 0;
 	else sp->stkoverflow = stkcur?stkcur->stkoverflow:overflow;
 	bsize = init+sizeof(struct frame);
-#ifndef USE_REALLOC
 	if(flags&STK_SMALL)
 		bsize = roundof(bsize,STK_FSIZE/16);
 	else
-#endif /* USE_REALLOC */
 		bsize = roundof(bsize,STK_FSIZE);
 	bsize -= sizeof(struct frame);
 	if(!(fp=newof((char*)0,struct frame, 1,bsize)))
@@ -279,9 +277,6 @@ Sfio_t *stkinstall(Sfio_t *stream, _stk_overflow_ oflow)
 		if(stream!=stkstd)
 			sfstack(stkstd,stream);
 		stkcur = sp;
-#ifdef USE_REALLOC
-		/*** someday ***/
-#endif /* USE_REALLOC */
 	}
 	else
 		sp = stkcur;
@@ -509,10 +504,8 @@ static char *stkgrow(register Sfio_t *stream, size_t size)
 	int nn=0,add=1;
 	n += (m + sizeof(struct frame)+1);
 	if(sp->stkflags&STK_SMALL)
-#ifndef USE_REALLOC
 		n = roundof(n,STK_FSIZE/16);
 	else
-#endif /* !USE_REALLOC */
 		n = roundof(n,STK_FSIZE);
 	/* see whether current frame can be extended */
 	if(stkptr(stream,0)==sp->stkbase+sizeof(struct frame))