Build scripts update (#20)

* Updated bromite base version, added new build scripts

* Sync dependencies on fresh checkout for bromite

* Remove unnecessary patch

README.md

@@ -15,14 +15,20 @@ How it works:
 	- Chromium build tools are set up in `./depot_tools`
 	- A Chromium source tree is set up in `./chromium` (this takes huge amounts of space)
 	- A `Bromite` source tree is set up in `./bromite` (this doesn't take much space)
-- Run `./patch_with_bromite.py` to checkout the correct version of Chromium and to apply the bromite patches to it
+- Run `./checkout_bromite_tag.py` to checkout the correct version of bromite to base the browser on
+	- Run this when the bromite version gets updated, you must then re-apply the bromite patches, it will auto-run at setup
+	- `bromite_tag.txt` is where you can update the bromite version to build on top of
+- Run `./apply_bromite_patches.py` to checkout the correct version of Chromium and to apply the bromite patches to it
   - Run this whenever bromite gets updated, it will auto-run at setup
+  - You will need to re-apply agregore patches next since the git state will be reset
 - Run `./apply_agregore_patches.py` to apply Agregore patches to the Chromium tree
   - Run this whenever there are new Agregore patches to apply. It will auto-run at setup
 - Run `./download_ipfs_daemon.py` to download the latest version of the [Agregore IPFS Daemon](https://github.com/AgregoreWeb/agregore-ipfs-daemon/)
 	- Run this whenever a new version is available to update. It will auto-run at setup
+	- The version of the daemon that we're using can be found inside `daemon_tag.txt`, update this with new versions
+	- Note: you can instead place an AAR of the daemon into `chromium/src/third_party/agregore-ipfs-daemon/agregore-ipfs-daemon.aar`
 - Run `./prebuild.py` to sync dependencies needed to perform a build.
-	- This can be skipped if you're just adding changes, dependenices can take an extra 30 GB of space
+	- This can be skipped if you're just adding changes, dependenices can take an extra 60 GB of space
 - Run `./build.py` to trigger a new build of the browser
   - You'll need to be running Ubuntu 18 in order to do a successful build.
   Automated builds via a [build server](https://build.mauve.moe) are a work in progress.

agregore_patch_order.txt

@@ -0,0 +1,3 @@
+0001-AG-Rebrand-to-Agregore.patch
+0001-AG-IPFS-Daemon.patch
+0001-AG-Handle-IPFS-and-IPNS-URLs.patch

apply_agregore_patches.py

@@ -21,29 +21,28 @@
 env = args["env"]
 
 patch_dir = os.path.join(root, 'patches')
+patch_list_file = os.path.join(root, "agregore_patch_order.txt")
 
 print("Applying patches")
-patch_list = os.listdir(patch_dir)
-patch_list.sort()
-
-for patch_name in patch_list:
-    raw_patch_path = os.path.join(patch_dir, patch_name)
-    patch_path = os.path.abspath(raw_patch_path)
-    print(f"Applying patch {patch_name}")
-    result = subprocess.run(
-        f"git am {patch_path}",
-        cwd=chromium,
-        shell=True, check=False,
-        stderr=PIPE, encoding="utf8"
-    )
-    if result.returncode != 0:
-        # If the patch was already applied, skip it. 🤷
-        if 'patch does not apply' in result.stderr:
-            print(f"Patch {patch_name} does not apply, skipping")
-            subprocess.run("git am --skip", cwd=chromium,
-                           shell=True, check=True)
-        else:
-            print(result.stderr)
-            result.check_returncode()
+with open(patch_list_file, 'r', encoding="utf8") as patch_list:
+    for patch_name_raw in patch_list:
+        patch_name = patch_name_raw.strip()
+        raw_patch_path = os.path.join(patch_dir, patch_name)
+        patch_path = os.path.abspath(raw_patch_path)
+        result = subprocess.run(
+            f"git am {patch_path}",
+            cwd=chromium,
+            shell=True, check=False,
+            stderr=PIPE, encoding="utf8"
+        )
+        if result.returncode != 0:
+            # If the patch was already applied, skip it. 🤷
+            if 'patch does not apply' in result.stderr:
+                print(f"Patch {patch_name} does not apply, skipping")
+                subprocess.run("git am --skip", cwd=chromium,
+                               shell=True, check=True)
+            else:
+                print(result.stderr)
+                result.check_returncode()
 
 print("Done!")

patch_with_bromite.py → apply_bromite_patches.py

@@ -8,6 +8,7 @@
 import os
 import argparse
 import subprocess
+from subprocess import PIPE
 import shutil
 from _load_vars import load_vars
 
@@ -18,8 +19,10 @@
 
 chromium = args["chromium"]
 bromite = args["bromite"]
+root = args["root"]
 build_name = args["build_name"]
 build_path = args["build_path"]
+depot_tools = args["depot_tools"]
 env = args["env"]
 
 bromite_build_folder = os.path.join(bromite, 'build')
@@ -30,10 +33,30 @@
 
 with open(release_file_name, 'r', encoding='utf8') as release_file:
     checkout_value = release_file.read()
+    print("Checking if commit already downloaded")
+    to_exec = f"git cat-file -t {checkout_value}"
+    result = subprocess.run(
+        to_exec, cwd=chromium,
+        shell=True, check=False, env=env,
+        stderr=PIPE, stdout=PIPE, encoding="utf8")
+    has_commit = "commit" in result.stdout
+
+    if has_commit:
+        print("Commit already downloaded")
+    else:
+        print(f"Fetching release commit {checkout_value}")
+        to_exec = f"git fetch --depth=1 origin {checkout_value}"
+        subprocess.run(to_exec, cwd=chromium, shell=True, check=True, env=env)
     print(f"Checking out: {checkout_value}")
     to_exec = f"git checkout {checkout_value}"
     subprocess.run(to_exec, cwd=chromium, shell=True, check=True, env=env)
 
+    if not has_commit:
+        out_folder = os.path.join(chromium, build_path)
+        if os.path.exists(out_folder):
+            print("Looks like first time checking out, syncing build tools")
+            subprocess.run('gclient sync -D', cwd=chromium, shell=True, check=True, env=env)
+
 # Copy build/bromite.gn_args to out/Default/args.gn (configurable?) (check if exists?)
 destination_gn_args = os.path.join(chromium, build_path, 'args.gn')
 if os.path.exists(destination_gn_args):
@@ -43,17 +66,30 @@
 else:
     print("Skipping args.gn since build folder was not initialized")
 
+print("Loading excluded patches list")
+PATCHES_TO_SKIP = ""
+patch_skip_list_file_name = os.path.join(root, "excluded_patches.txt")
+with open(patch_skip_list_file_name, 'r', encoding="utf8") as patch_skip_file:
+    PATCHES_TO_SKIP = patch_skip_file.read()
+
 # Get `bromite/build/bromite_patches_list.txt` and read each line
+# Skip patches identified in "excluded_patches.txt"
 print("Applying patches")
 patch_list_file_name = os.path.join(
     bromite_build_folder, 'bromite_patches_list.txt')
 with open(patch_list_file_name, 'r', encoding='utf8') as patch_list:
-    for patch_name in patch_list:
+    for patch_name_raw in patch_list:
+        patch_name = patch_name_raw.strip()
+
+        if patch_name in PATCHES_TO_SKIP:
+            print(f"Skipping: {patch_name}")
+            continue
+
         raw_patch_path = os.path.join(
             bromite_build_folder, 'patches', patch_name.strip())
         patch_path = os.path.abspath(raw_patch_path)
+
         # Use `git am` to apply patches one by one from `bromite/build/patches/`
-        print(f"Applying patch {patch_name}")
         subprocess.run(f"git am {patch_path}",
                        cwd=chromium, shell=True, check=True)
 

bromite_tag.txt

@@ -0,0 +1 @@
+100.0.4896.135

checkout_bromite_tag.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+
+'''
+This script will checkout the version of bromite specified in bromite_tag.txt
+'''
+
+import os
+import subprocess
+import argparse
+
+from _load_vars import load_vars
+
+parser = argparse.ArgumentParser(
+    description='Checkout the version of bromite specified in bromite_tag.txt'
+)
+
+bromite_tag_file_path = os.path.normpath(os.path.join(
+    os.path.realpath(__file__),
+    "../bromite_tag.txt"
+))
+
+DEFAULT_VERSION = ""
+
+with open(bromite_tag_file_path, "r", encoding="utf8") as bromite_tag_file:
+    DEFAULT_VERSION = bromite_tag_file.read().strip()
+
+parser.add_argument(
+    '--version',
+    default=DEFAULT_VERSION,
+    help="What version tag to use for bromite"
+)
+
+args = load_vars(parser)
+
+env = args["env"]
+bromite = args["bromite"]
+
+version = args["version"]
+
+print(f"Checking out bromite version {version}")
+
+to_exec = f"git checkout {version}"
+subprocess.run(to_exec, cwd=bromite, shell=True, check=True, env=env)
+
+print("Done!")

daemon_tag.txt

@@ -0,0 +1 @@
+zeroconf-fix-2

download_ipfs_daemon.py

@@ -14,8 +14,16 @@
     description='Download the IPFS daemon AAR file'
 )
 
-# TODO: Use specific tag once we have a stable release
-DEFAULT_VERSION = "latest"
+daemon_tag_file_path = os.path.normpath(os.path.join(
+    os.path.realpath(__file__),
+    "../daemon_tag.txt"
+))
+
+DEFAULT_VERSION = ""
+
+with open(daemon_tag_file_path, "r", encoding="utf8") as daemon_tag_file:
+    DEFAULT_VERSION = daemon_tag_file.read().strip()
+
 DEFAULT_REPO = 'AgregoreWeb/agregore-ipfs-daemon'
 DEFAULT_BINARY_NAME = 'agregore-ipfs-daemon.aar'
 
@@ -46,12 +54,15 @@
 repo = args["repo"]
 binary_name = args["binary_name"]
 
-url = f"https://github.com/{repo}/releases/{version}/download/{binary_name}"
+url = f"https://github.com/{repo}/releases/download/{version}/{binary_name}"
 
 print(f"Downloading {url}")
 
 response = requests.get(url, allow_redirects=True, stream=True)
 
+# Raise an exception if we get a 404
+response.raise_for_status()
+
 download_location = os.path.join(
     chromium,
     "third_party/agregore-ipfs-daemon/agregore-ipfs-daemon.aar"

excluded_patches.txt

@@ -0,0 +1,4 @@
+Bromite-auto-updater.patch
+Bromite-package-name.patch
+# Not sure why, but this patch keeps breaking
+Disable-minidump-upload-scheduling.patch

generate_patch.py

@@ -15,9 +15,15 @@
 )
 
 parser.add_argument(
-    '--n',
-    default="HEAD^",
-    help="Number of commits to use for the patch"
+    '--commit',
+    default="HEAD",
+    help="The commit to base the patch off of"
+)
+
+parser.add_argument(
+    "--n",
+    default="1",
+    help="The number of commits to include in the patch"
 )
 
 args = load_vars(parser)
@@ -26,15 +32,15 @@
 env = args["env"]
 root = args["root"]
 n = args["n"]
+commit = args["commit"]
 
 patch_dir = os.path.join(root, 'patches')
 
 print("Generating patch from latest commit")
-to_run = f'git format-patch {n} -o {patch_dir}'
+to_run = f'git format-patch --no-numbered -{n} {commit} -o {patch_dir}'
 result = subprocess.run(to_run, cwd=chromium, shell=True,
                         check=True, env=env, encoding="utf8")
 
 patch_name = result.stdout
 
-
 print("Done!")