GFF_Annotater.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1460895a-baf3-4fc9-938c-0979a0679ca1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "####Packages####\n",
    "from Bio import SeqIO\n",
    "from BCBio.GFF import GFFExaminer\n",
    "from Bio.Seq import Seq\n",
    "from Bio.SeqFeature import SeqFeature, FeatureLocation\n",
    "from Bio.SeqRecord import SeqRecord\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gff3_parser\n",
    "import glob\n",
    "import numpy as np\n",
    "import os\n",
    "from pathlib import Path\n",
    "import pprint\n",
    "import re\n",
    "import shutil\n",
    "import subprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "eee9c0bf-5fb7-49fa-877a-ea12414f0e7b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Anguilla_anguilla Genome imported\n",
      "There are 22 total hsp genes found in genome\n",
      "8 hsp genes are found on the negative strand\n",
      "14 hsp genes are found on the positive strand\n"
     ]
    }
   ],
   "source": [
    "#Pull in the Genome Annotation File for the correct species & Parse\n",
    "os.chdir('/home/jake/Downloads/Genomes/gff_files/')\n",
    "anotes=os.listdir('/home/jake/Downloads/Genomes/gff_files/')\n",
    "for f in anotes:\n",
    "    gff=os.path.basename(f)\n",
    "    name=gff.split('.')[0] #Removes.gff\n",
    "    gname=name\n",
    "    _ = gff3_parser.parse_gff3(gff,verbose = False,  parse_attributes=True)\n",
    "    df=_[['Seqid', 'Type', 'Start', 'End', 'Strand','gene', 'gene_biotype']]\n",
    "    df.index.name='Index'\n",
    "    df=df.loc[(df['Type']=='gene')& (df['gene_biotype']=='protein_coding')]\n",
    "    df=df.astype({'Start':int,'End':int})\n",
    "    #Create Filtered Gene Dataframes\n",
    "    all_df=df.loc[(df['Type']=='gene') & (df['gene'].str.contains(r'hsp', re.I))] # ALL HSP genes\n",
    "    neg_df=df.loc[(df['Strand']=='-')& (df['Type']=='gene') & (df['gene'].str.contains(r'hsp', re.I))] #Negative HSP\n",
    "    pos_df=df.loc[(df['Strand']=='+')& (df['Type']=='gene') & (df['gene'].str.contains(r'hsp', re.I))] #Positive HSP\n",
    "    alen=len(all_df)\n",
    "    nlen=len(neg_df)\n",
    "    plen=len(pos_df)\n",
    "    print(f\"{name} Genome imported\")\n",
    "    print(f\"There are {alen} total hsp genes found in genome\")\n",
    "    print(f\"{nlen} hsp genes are found on the negative strand\")\n",
    "    print(f\"{plen} hsp genes are found on the positive strand\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1c14f9f8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Negative Strands Created Sucessfully\n",
      "Positive Strands Created Sucessfully\n"
     ]
    }
   ],
   "source": [
    "#Annotated Gene Strands DF's\n",
    "nstrand=df.loc[(df['Strand']=='-')].sort_values(by=\"Start\", ascending=True)#Negative Strand Genes\n",
    "#master_nstrand=df.loc[(df['Strand']=='-')].sort_values(by=\"Start\", ascending=True)\n",
    "pstrand=df.loc[(df['Strand']=='+')].sort_values(by=\"Start\", ascending=True)#Negative Strand Genes\n",
    "#List of Columns to manipulate\n",
    "clist=['shift_1', 'shift_2', 'shift_3', 'shift_4', 'shift_5', 'shift_6', 'shift_7', 'shift_8', 'shift_9', 'shift_10']\n",
    "print(f\"Negative Strands Created Sucessfully\")\n",
    "print(f\"Positive Strands Created Sucessfully\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7710ff12",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Negative Strand Manipulated Sucessfully\n",
      "Positive Strand Manipulated Sucessfully\n",
      "Chromosome Column Created\n"
     ]
    }
   ],
   "source": [
    "#Function  manipulate the dataframes\n",
    "def seq_manipulate(df):\n",
    "    df['shift_1']=df['Start']- df['End'].shift(+1)\n",
    "    df['shift_2']=df['Start']- df['End'].shift(+2)\n",
    "    df['shift_3']=df['Start']- df['End'].shift(+3)\n",
    "    df['shift_4']=df['Start']- df['End'].shift(+4)\n",
    "    df['shift_5']=df['Start']- df['End'].shift(+5)\n",
    "    df['shift_6']=df['Start']- df['End'].shift(+6)\n",
    "    df['shift_7']=df['Start']- df['End'].shift(+7)\n",
    "    df['shift_8']=df['Start']- df['End'].shift(+8)\n",
    "    df['shift_9']=df['Start']- df['End'].shift(+9)\n",
    "    df['shift_10']=df['Start']- df['End'].shift(+10)\n",
    "    df[clist]= df[clist].clip(lower=0)#Remove all of the negative values and convert them to 0\n",
    "    df[clist]= df[clist].fillna(0) #Convert NA values to 0\n",
    "    df[clist]= df[clist].replace(0, np.nan) #Convert all the 0 values to NA\n",
    "    df['inter_genic']=df[clist].min(1)\n",
    "    df['Stop']= df['Start'] + df['inter_genic']\n",
    "    df['combined']=df.apply(lambda x:'%s:%s'%(x['Start'],x['End']),axis=1)\n",
    "    #df.update('\"' + df['combined'].astype(str) + '\"')\n",
    "#Transfrom both strands of DFs\n",
    "\n",
    "seq_manipulate(nstrand)\n",
    "print(f\"Negative Strand Manipulated Sucessfully\")\n",
    "seq_manipulate(pstrand)\n",
    "print(f\"Positive Strand Manipulated Sucessfully\")\n",
    "#Create Two Filtered Data Sets\n",
    "n_=nstrand[['Seqid', 'Type', 'Strand','gene', 'gene_biotype', 'Start', 'Stop', 'combined']]\n",
    "p_=pstrand[['Seqid', 'Type', 'Strand','gene', 'gene_biotype', 'Start', 'Stop', 'combined']]\n",
    "n_hsp=n_.loc[(n_['gene'].str.contains(r'hsp', re.I))] # ALL HSP genes\n",
    "p_hsp=p_.loc[(p_['gene'].str.contains(r'hsp', re.I))] # ALL HSP genes\n",
    "hsps= pd.concat([n_hsp,p_hsp])\n",
    "hsps=hsps.sort_values(by=\"Start\", ascending=True)\n",
    "hsps['Chromosome']= hsps['Seqid'].apply(lambda x: x[-4:-2])\n",
    "export=hsps[['gene', 'combined', 'Chromosome', 'Start', 'Stop']]\n",
    "x=export.reset_index()\n",
    "x=x[['gene', 'combined', 'Chromosome', 'Start', 'Stop']]\n",
    "x.to_csv(f\"/home/jake/Downloads/{name}.csv\")\n",
    "print(f\"Chromosome Column Created\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "045e2704",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "copied chr10.fna\n",
      "copied chr11.fna\n",
      "copied chr12.fna\n",
      "copied chr13.fna\n",
      "copied chr14.fna\n",
      "copied chr15.fna\n",
      "copied chr16.fna\n",
      "copied chr17.fna\n",
      "copied chr18.fna\n",
      "copied chr19.fna\n",
      "copied chr01.fna\n",
      "copied chr02.fna\n",
      "copied chr03.fna\n",
      "copied chr04.fna\n",
      "copied chr05.fna\n",
      "copied chr06.fna\n",
      "copied chr07.fna\n",
      "copied chr08.fna\n",
      "copied chr09.fna\n",
      "Now open terminal, cd into downloads and run ./for_loop_updated.sh for Anguilla_anguilla\n"
     ]
    }
   ],
   "source": [
    "#Need to copy the genome directory and remove the line wraps(/n)\n",
    "source_folder=(f\"/home/jake/Downloads/Genomes/{name}/\")\n",
    "destination_folder= (f\"/home/jake/Downloads/Genomes/{name}_noninterleaved/\")\n",
    "path = (f\"/home/jake/Downloads/Genomes/{name}_noninterleaved/\")\n",
    "# Check whether the specified path exists or not\n",
    "isExist = os.path.exists(path)\n",
    "if not isExist:\n",
    "  # Create a new directory because it does not exist \n",
    "  os.makedirs(path)\n",
    "  print(\"The new directory is created!\")\n",
    "for file_name in os.listdir(source_folder):\n",
    "    # construct full file path\n",
    "    source = source_folder + file_name\n",
    "    destination = destination_folder + file_name\n",
    "    # copy only files\n",
    "    if os.path.isfile(source):\n",
    "        shutil.copy(source, destination)\n",
    "        print('copied', file_name)\n",
    "\n",
    "print(f\"Now open terminal, cd into downloads and run ./for_loop_updated.sh for {name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e811847e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Going to attempt to read in each sequence, parse it into a filename and seq:\n",
    "folder_path = path\n",
    "fasta_paths = glob.glob(os.path.join(folder_path, '*.fna'))\n",
    "seq=[]\n",
    "names=[]\n",
    "for fasta_path in fasta_paths:\n",
    "    names.append(os.path.basename(fasta_path))\n",
    "    for seq_record in SeqIO.parse(fasta_path, \"fasta\"):\n",
    "        seq.append(seq_record.seq)\n",
    "fasta_dict={names[i]:seq[i] for i in range(len(names))}\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8e45f56d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Itterate through the df\n",
    "slice= []\n",
    "for f in x['Chromosome']:\n",
    "    for name in fasta_dict:\n",
    "        if f in name:\n",
    "            value_pair=(fasta_dict.get(name))\n",
    "            slice.append(value_pair)\n",
    "            #print(value_pair[0:10])\n",
    "            x['Chr']=value_pair\n",
    "x['Stop'] = x['Stop'].astype(int)\n",
    "x['new_col'] = x.apply(lambda x: x['Chr'][x['Start']-1:x['Stop']], 1)\n",
    "#x['new_col']=str(seq(x['new_col']))\n",
    "x['len']= x['new_col'].str.len()\n",
    "x['header']=x['gene']\n",
    "we_did_it=dict(zip(x[\"header\"], x[\"new_col\"]))\n",
    "x['new_col']=x['new_col'].astype(str)\n",
    "#display(x['new_col'])\n",
    "formatted=[]\n",
    "for i in we_did_it.values():\n",
    "    v = str(i)\n",
    "    formatted.append(v)\n",
    "x['format']=formatted\n",
    "fml=dict(zip(x[\"header\"], x[\"format\"]))\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "452c76f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exported fasta sequence file is in /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla\n"
     ]
    }
   ],
   "source": [
    "destination_folder= (f\"/home/jake/Downloads/Genomes/intergenic_exports/{gname}\")\n",
    "path = (f\"/home/jake/Downloads/Genomes/intergenic_exports/{gname}\")\n",
    "# Check whether the specified path exists or not\n",
    "isExist = os.path.exists(path)\n",
    "if not isExist:\n",
    "  # Create a new directory because it does not exist \n",
    "  os.makedirs(path)\n",
    "  print(\"The new directory is created!\")\n",
    "os.chdir(path)\n",
    "with open(f'{gname}_clean.fasta', 'w') as output_file:\n",
    "    for name, seq in fml.items():\n",
    "        output_file.write(f\">{gname}_{name}\\n{seq}\\n\")\n",
    "\n",
    "print(f\"Exported fasta sequence file is in {destination_folder}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "ec49fa6a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla\n"
     ]
    }
   ],
   "source": [
    "print(os.getcwd())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "542d8251",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Open Command line and input this command\n",
    "##\n",
    "##cat ______clean.fasta | awk '{\n",
    "        if (substr($0, 1, 1)==\">\") {filename=(substr($0,2) \".fasta\")}\n",
    "        print $0 >> filename\n",
    "        close(filename)\n",
    "}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "2dc3629e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_clean.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 2\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 2\n",
      "identifying Simple Repeats in batch 1 of 2\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 2 of 2\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 2 of 2\n",
      "identifying Simple Repeats in batch 2 of 2\n",
      "processing output: \n",
      "cycle 1 \n",
      "cycle 2 \n",
      "cycle 3 \n",
      "cycle 4 \n",
      "cycle 5 \n",
      "cycle 6 \n",
      "cycle 7 \n",
      "cycle 8 \n",
      "cycle 9 \n",
      "cycle 10 \n",
      "Generating output... \n",
      "masking\n",
      "done\n",
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspa12a.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 1\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "processing output: \n",
      "cycle 1 \n",
      "cycle 2 \n",
      "cycle 3 \n",
      "cycle 4 \n",
      "cycle 5 \n",
      "cycle 6 \n",
      "cycle 7 \n",
      "cycle 8 \n",
      "cycle 9 \n",
      "cycle 10 \n",
      "Generating output... \n",
      "masking\n",
      "done\n",
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspa4a.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 1\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "processing output: \n",
      "cycle 1 \n",
      "cycle 2 \n",
      "cycle 3 \n",
      "cycle 4 \n",
      "cycle 5 \n",
      "cycle 6 \n",
      "cycle 7 \n",
      "cycle 8 \n",
      "cycle 9 \n",
      "cycle 10 \n",
      "Generating output... \n",
      "masking\n",
      "done\n",
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspb8.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 1\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "processing output: \n",
      "cycle 1 \n",
      "cycle 2 \n",
      "cycle 3 \n",
      "cycle 4 \n",
      "cycle 5 \n",
      "cycle 6 \n",
      "cycle 7 \n",
      "cycle 8 \n",
      "cycle 9 \n",
      "cycle 10 \n",
      "Generating output... \n",
      "masking\n",
      "done\n",
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hsp90b1.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 1\n",
      "processing output: \n",
      "cycle 1 \n",
      "cycle 2 \n",
      "cycle 3 \n",
      "cycle 4 \n",
      "cycle 5 \n",
      "cycle 6 \n",
      "cycle 7 \n",
      "cycle 8 \n",
      "cycle 9 \n",
      "cycle 10 \n",
      "Generating output... \n",
      "masking\n",
      "done\n",
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspb7.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 1\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "processing output: \n",
      "cycle 1 \n",
      "cycle 2 \n",
      "cycle 3 \n",
      "cycle 4 \n",
      "cycle 5 \n",
      "cycle 6 \n",
      "cycle 7 \n",
      "cycle 8 \n",
      "cycle 9 \n",
      "cycle 10 \n",
      "Generating output... \n",
      "masking\n",
      "done\n",
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspa13.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 1\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "\n",
      "\n",
      "\n",
      "No repetitive sequences were detected in /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspa13.fasta\n",
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_carhsp1.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 1\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "processing output: \n",
      "cycle 1 \n",
      "cycle 2 \n",
      "cycle 3 \n",
      "cycle 4 \n",
      "cycle 5 \n",
      "cycle 6 \n",
      "cycle 7 \n",
      "cycle 8 \n",
      "cycle 9 \n",
      "cycle 10 \n",
      "Generating output... \n",
      "masking\n",
      "done\n",
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspa14.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 1\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "\n",
      "\n",
      "\n",
      "No repetitive sequences were detected in /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspa14.fasta\n",
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspbp1.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 1\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "\n",
      "\n",
      "\n",
      "No repetitive sequences were detected in /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspbp1.fasta\n",
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspb9.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 1\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "\n",
      "\n",
      "\n",
      "No repetitive sequences were detected in /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspb9.fasta\n",
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspb2.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 1\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "processing output: \n",
      "cycle 1 \n",
      "cycle 2 \n",
      "cycle 3 \n",
      "cycle 4 \n",
      "cycle 5 \n",
      "cycle 6 \n",
      "cycle 7 \n",
      "cycle 8 \n",
      "cycle 9 \n",
      "cycle 10 \n",
      "Generating output... \n",
      "masking\n",
      "done\n",
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspa12b.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 1\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "processing output: \n",
      "cycle 1 \n",
      "cycle 2 \n",
      "cycle 3 \n",
      "cycle 4 \n",
      "cycle 5 \n",
      "cycle 6 \n",
      "cycle 7 \n",
      "cycle 8 \n",
      "cycle 9 \n",
      "cycle 10 \n",
      "Generating output... \n",
      "masking\n",
      "done\n",
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspa8.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 1\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "processing output: \n",
      "cycle 1 \n",
      "cycle 2 \n",
      "cycle 3 \n",
      "cycle 4 \n",
      "cycle 5 \n",
      "cycle 6 \n",
      "cycle 7 \n",
      "cycle 8 \n",
      "cycle 9 \n",
      "cycle 10 \n",
      "Generating output... \n",
      "masking\n",
      "done\n",
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hsph1.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 1\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "processing output: \n",
      "cycle 1 \n",
      "cycle 2 \n",
      "cycle 3 \n",
      "cycle 4 \n",
      "cycle 5 \n",
      "cycle 6 \n",
      "cycle 7 \n",
      "cycle 8 \n",
      "cycle 9 \n",
      "cycle 10 \n",
      "Generating output... \n",
      "masking\n",
      "done\n",
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspb3.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 1\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "processing output: \n",
      "cycle 1 \n",
      "cycle 2 \n",
      "cycle 3 \n",
      "cycle 4 \n",
      "cycle 5 \n",
      "cycle 6 \n",
      "cycle 7 \n",
      "cycle 8 \n",
      "cycle 9 \n",
      "cycle 10 \n",
      "Generating output... \n",
      "masking\n",
      "done\n",
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspa1b.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 1\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "processing output: \n",
      "cycle 1 \n",
      "cycle 2 \n",
      "cycle 3 \n",
      "cycle 4 \n",
      "cycle 5 \n",
      "cycle 6 \n",
      "cycle 7 \n",
      "cycle 8 \n",
      "cycle 9 \n",
      "cycle 10 \n",
      "Generating output... \n",
      "masking\n",
      "done\n",
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspb1.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 1\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "processing output: \n",
      "cycle 1 \n",
      "cycle 2 \n",
      "cycle 3 \n",
      "cycle 4 \n",
      "cycle 5 \n",
      "cycle 6 \n",
      "cycle 7 \n",
      "cycle 8 \n",
      "cycle 9 \n",
      "cycle 10 \n",
      "Generating output... \n",
      "masking\n",
      "done\n",
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hsp90ab1.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 1\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "processing output: \n",
      "cycle 1 \n",
      "cycle 2 \n",
      "cycle 3 \n",
      "cycle 4 \n",
      "cycle 5 \n",
      "cycle 6 \n",
      "cycle 7 \n",
      "cycle 8 \n",
      "cycle 9 \n",
      "cycle 10 \n",
      "Generating output... \n",
      "masking\n",
      "done\n",
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspa9.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 1\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "processing output: \n",
      "cycle 1 \n",
      "cycle 2 \n",
      "cycle 3 \n",
      "cycle 4 \n",
      "cycle 5 \n",
      "cycle 6 \n",
      "cycle 7 \n",
      "cycle 8 \n",
      "cycle 9 \n",
      "cycle 10 \n",
      "Generating output... \n",
      "masking\n",
      "done\n",
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspd1.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 1\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "processing output: \n",
      "cycle 1 \n",
      "cycle 2 \n",
      "cycle 3 \n",
      "cycle 4 \n",
      "cycle 5 \n",
      "cycle 6 \n",
      "cycle 7 \n",
      "cycle 8 \n",
      "cycle 9 \n",
      "cycle 10 \n",
      "Generating output... \n",
      "masking\n",
      "done\n",
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspbap1.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 1\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "processing output: \n",
      "cycle 1 \n",
      "cycle 2 \n",
      "cycle 3 \n",
      "cycle 4 \n",
      "cycle 5 \n",
      "cycle 6 \n",
      "cycle 7 \n",
      "cycle 8 \n",
      "cycle 9 \n",
      "cycle 10 \n",
      "Generating output... \n",
      "masking\n",
      "done\n",
      "RepeatMasker version 4.1.2-p1\n",
      "Search Engine: NCBI/RMBLAST [ 2.11.0+ ]\n",
      "Using Custom Repeat Library: /home/jake/Downloads/Genomes/FishTEDB/Anguilla_anguilla_lib.fa\n",
      "\n",
      "\n",
      "analyzing file /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspb6.fasta\n",
      "\n",
      "Checking for E. coli insertion elements\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "identifying matches to Anguilla_anguilla_lib.fa sequences in batch 1 of 1\n",
      "identifying Simple Repeats in batch 1 of 1\n",
      "\n",
      "\n",
      "\n",
      "No repetitive sequences were detected in /home/jake/Downloads/Genomes/intergenic_exports/Anguilla_anguilla/Anguilla_anguilla_hspb6.fasta\n"
     ]
    }
   ],
   "source": [
    "##Change directory into repeat masker and run the script\n",
    "lib_path='/home/jake/Downloads/Genomes/FishTEDB/'\n",
    "lib=(f\"'/home/jake/Downloads/Genomes/FishTEDB/'{gname}_lib.fa\")\n",
    "#See GFF_Annotater for the genome and gene variables\n",
    "destz=(f'/home/jake/Downloads/Genomes/intergenic_exports/{gname}/')\n",
    "os.chdir('/home/jake/Downloads/Bio_tools/RepeatMasker-4.1.2-p1/RepeatMasker/')\n",
    "anotez=os.listdir(f'/home/jake/Downloads/Genomes/intergenic_exports/{gname}/')\n",
    "for w in anotez:\n",
    "    proximal_promoter=destz+w\n",
    "    os.system(f\"./RepeatMasker -lib {lib} {proximal_promoter}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "c3f0ef72",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Moved:  Anguilla_anguilla_carhsp1.fasta.cat\n",
      "Moved:  Anguilla_anguilla_carhsp1.fasta.masked\n",
      "Moved:  Anguilla_anguilla_carhsp1.fasta.out\n",
      "Moved:  Anguilla_anguilla_carhsp1.fasta.tbl\n",
      "Moved:  Anguilla_anguilla_clean.fasta.cat\n",
      "Moved:  Anguilla_anguilla_clean.fasta.masked\n",
      "Moved:  Anguilla_anguilla_clean.fasta.out\n",
      "Moved:  Anguilla_anguilla_clean.fasta.tbl\n",
      "Moved:  Anguilla_anguilla_hsp90ab1.fasta.cat\n",
      "Moved:  Anguilla_anguilla_hsp90ab1.fasta.masked\n",
      "Moved:  Anguilla_anguilla_hsp90ab1.fasta.out\n",
      "Moved:  Anguilla_anguilla_hsp90ab1.fasta.tbl\n",
      "Moved:  Anguilla_anguilla_hsp90b1.fasta.cat\n",
      "Moved:  Anguilla_anguilla_hsp90b1.fasta.masked\n",
      "Moved:  Anguilla_anguilla_hsp90b1.fasta.out\n",
      "Moved:  Anguilla_anguilla_hsp90b1.fasta.tbl\n",
      "Moved:  Anguilla_anguilla_hspa1b.fasta.cat\n",
      "Moved:  Anguilla_anguilla_hspa1b.fasta.masked\n",
      "Moved:  Anguilla_anguilla_hspa1b.fasta.out\n",
      "Moved:  Anguilla_anguilla_hspa1b.fasta.tbl\n",
      "Moved:  Anguilla_anguilla_hspa4a.fasta.cat\n",
      "Moved:  Anguilla_anguilla_hspa4a.fasta.masked\n",
      "Moved:  Anguilla_anguilla_hspa4a.fasta.out\n",
      "Moved:  Anguilla_anguilla_hspa4a.fasta.tbl\n",
      "Moved:  Anguilla_anguilla_hspa8.fasta.cat\n",
      "Moved:  Anguilla_anguilla_hspa8.fasta.masked\n",
      "Moved:  Anguilla_anguilla_hspa8.fasta.out\n",
      "Moved:  Anguilla_anguilla_hspa8.fasta.tbl\n",
      "Moved:  Anguilla_anguilla_hspa9.fasta.cat\n",
      "Moved:  Anguilla_anguilla_hspa9.fasta.masked\n",
      "Moved:  Anguilla_anguilla_hspa9.fasta.out\n",
      "Moved:  Anguilla_anguilla_hspa9.fasta.tbl\n",
      "Moved:  Anguilla_anguilla_hspa12a.fasta.cat\n",
      "Moved:  Anguilla_anguilla_hspa12a.fasta.masked\n",
      "Moved:  Anguilla_anguilla_hspa12a.fasta.out\n",
      "Moved:  Anguilla_anguilla_hspa12a.fasta.tbl\n",
      "Moved:  Anguilla_anguilla_hspa12b.fasta.cat\n",
      "Moved:  Anguilla_anguilla_hspa12b.fasta.masked\n",
      "Moved:  Anguilla_anguilla_hspa12b.fasta.out\n",
      "Moved:  Anguilla_anguilla_hspa12b.fasta.tbl\n",
      "Moved:  Anguilla_anguilla_hspa13.fasta.cat\n",
      "Moved:  Anguilla_anguilla_hspa13.fasta.cat.all\n",
      "Moved:  Anguilla_anguilla_hspa13.fasta.out\n",
      "Moved:  Anguilla_anguilla_hspa14.fasta.cat\n",
      "Moved:  Anguilla_anguilla_hspa14.fasta.cat.all\n",
      "Moved:  Anguilla_anguilla_hspa14.fasta.out\n",
      "Moved:  Anguilla_anguilla_hspb1.fasta.cat\n",
      "Moved:  Anguilla_anguilla_hspb1.fasta.masked\n",
      "Moved:  Anguilla_anguilla_hspb1.fasta.out\n",
      "Moved:  Anguilla_anguilla_hspb1.fasta.tbl\n",
      "Moved:  Anguilla_anguilla_hspb2.fasta.cat\n",
      "Moved:  Anguilla_anguilla_hspb2.fasta.masked\n",
      "Moved:  Anguilla_anguilla_hspb2.fasta.out\n",
      "Moved:  Anguilla_anguilla_hspb2.fasta.tbl\n",
      "Moved:  Anguilla_anguilla_hspb3.fasta.cat\n",
      "Moved:  Anguilla_anguilla_hspb3.fasta.masked\n",
      "Moved:  Anguilla_anguilla_hspb3.fasta.out\n",
      "Moved:  Anguilla_anguilla_hspb3.fasta.tbl\n",
      "Moved:  Anguilla_anguilla_hspb6.fasta.cat\n",
      "Moved:  Anguilla_anguilla_hspb6.fasta.cat.all\n",
      "Moved:  Anguilla_anguilla_hspb6.fasta.out\n",
      "Moved:  Anguilla_anguilla_hspb7.fasta.cat\n",
      "Moved:  Anguilla_anguilla_hspb7.fasta.masked\n",
      "Moved:  Anguilla_anguilla_hspb7.fasta.out\n",
      "Moved:  Anguilla_anguilla_hspb7.fasta.tbl\n",
      "Moved:  Anguilla_anguilla_hspb8.fasta.cat\n",
      "Moved:  Anguilla_anguilla_hspb8.fasta.masked\n",
      "Moved:  Anguilla_anguilla_hspb8.fasta.out\n",
      "Moved:  Anguilla_anguilla_hspb8.fasta.tbl\n",
      "Moved:  Anguilla_anguilla_hspb9.fasta.cat\n",
      "Moved:  Anguilla_anguilla_hspb9.fasta.cat.all\n",
      "Moved:  Anguilla_anguilla_hspb9.fasta.out\n",
      "Moved:  Anguilla_anguilla_hspbap1.fasta.cat\n",
      "Moved:  Anguilla_anguilla_hspbap1.fasta.masked\n",
      "Moved:  Anguilla_anguilla_hspbap1.fasta.out\n",
      "Moved:  Anguilla_anguilla_hspbap1.fasta.tbl\n",
      "Moved:  Anguilla_anguilla_hspbp1.fasta.cat\n",
      "Moved:  Anguilla_anguilla_hspbp1.fasta.cat.all\n",
      "Moved:  Anguilla_anguilla_hspbp1.fasta.out\n",
      "Moved:  Anguilla_anguilla_hspd1.fasta.cat\n",
      "Moved:  Anguilla_anguilla_hspd1.fasta.masked\n",
      "Moved:  Anguilla_anguilla_hspd1.fasta.out\n",
      "Moved:  Anguilla_anguilla_hspd1.fasta.tbl\n",
      "Moved:  Anguilla_anguilla_hsph1.fasta.cat\n",
      "Moved:  Anguilla_anguilla_hsph1.fasta.masked\n",
      "Moved:  Anguilla_anguilla_hsph1.fasta.out\n",
      "Moved:  Anguilla_anguilla_hsph1.fasta.tbl\n"
     ]
    }
   ],
   "source": [
    "source_folderz = (f'/home/jake/Downloads/Genomes/intergenic_exports/{gname}/')\n",
    "destination_folderz = (f'/home/jake/Downloads/repeat_outputs/{gname}/')\n",
    "isExist = os.path.exists(destination_folderz)\n",
    "if not isExist:\n",
    "  # Create a new directory because it does not exist \n",
    "  os.makedirs(destination_folderz)\n",
    "  print(\"The new directory is created!\")\n",
    "outputz=os.listdir(source_folderz)\n",
    "# iterate files\n",
    "for f in outputz:\n",
    "    if f.endswith(('cat','tbl','all','masked','out')):\n",
    "        src=source_folderz+f\n",
    "        dest=destination_folderz+f\n",
    "        shutil.move(src, dest)\n",
    "        print(\"Moved: \", f)\n",
    "    else:\n",
    "        pass"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "vscode": {
   "interpreter": {
    "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}