Merge pull request #8 from mle2718/update2024

Update2024

documentation/data_update_documentation.md

@@ -8,21 +8,21 @@ MRIP data is stored at ``\\net\mrfss.``  The file ``copy_over_raw_mrip.do`` assu
 ## Stat-tranfser
 Ensure that your stat-transfer executable can be found. It is set with in profile.do
 
+
+# Files that need to be run
+1. ``extraction_wrapper.do'' and ``process_wrapper.do'' are the only files that need to be run
+
+
 # Files that need to be changed
 
 1.  ``extraction_wrapper.do``: Adjust the full years and partial years.  Also adjust the wavelist if needed. We always prototype  the model on wave 4 data and then finish running it when the wave 5 data is released.
-2.  ``batch_file_to_process_monthly_mrip_data.do``- we always run on the most recent 2 years of data, so the numlist containing years should be updated.  May have to update the 2b95 hack.
-3. ``convert_monthly_to_annual.do`` and ``subset_monthly_mrip`` - adjust the BLAST_DIR and yr1 locals. This is a loop-by-hand. If it is 2022, you need to set ``yr1=2021`` and then ``yr1==2022``
 4. ``catch_summaries.txt`` - This is a data summary dynamic document in stata. I use it to print out the trips into a nice document.  You should only need to change  the ``vintage_string`` and ``this_year` globals.  You may need to deal with landings_old 
 
 
 # Files that shouldn't need to be changed
 1. ``batch_file_to_process_annual_mrip_data.do`` - not using an annual timestep.
 2. ``copy_over_raw_mrip.do``  -- this just copies file.  
-
-# Files that need to be run
-1. ``extraction_wrapper.do'' and ``process_wrapper.do`` are the only files that need to be run
-
+3. ``convert_monthly_to_annual.do`` and ``subset_monthly_mrip``
 
 # Notes from 2022
 
@@ -37,3 +37,21 @@ The code assumes that the units from the historical numbers-at-age are in 000s o
 
 1. Constructing the historical selectivity.
 2. Initial age structures.
+
+# Notes from 2023
+
+Normally, we do this:
+```
+svyset psu_id [pweight= wp_size], strata(var_id) singleunit(certainty)
+svy: tab l_in_bin my_dom_id_string, count
+
+```
+
+We do not use the survey weights to compute the length distribution of B2 cod.  This is consistent with the stock assessment.
+
+For B2 cod, we do this instead
+```
+svyset psu_id,  strata(var_id) singleunit(certainty)
+svy: tab l_in_bin my_dom_id_string, count
+
+```

stata_code/data_extraction_processing/extraction/get_ma_allocation.do

@@ -2,6 +2,6 @@
 /* read in and store the MRIP location stuff.  Put it in the file ma_site_allocation.dta */
 clear
 #delimit;
-odbc load,  exec("select site_id, stock_region_calc from mlee.mrip_ma_site_list;") $mysole_conn;
+odbc load,  exec("select site_id, stock_region_calc from recdbs.mrip_ma_site_list;") $myNEFSC_USERS_conn;
 #delimit cr
 save "${data_raw}/ma_site_allocation.dta", replace

stata_code/data_extraction_processing/extraction_wrapper.do

@@ -4,17 +4,15 @@
 *do ${extraction_code}/get_ma_allocation.do
 
 
-/* copying over full years of data */
-global yearlist 2019(1)2021
-global wavelist 1(1)6
-
-do ${extraction_code}/copy_over_raw_mrip.do
-
-
-
 /* copying over partial years of data */
-global yearlist 2022(1)2022
-global wavelist 1(1)5
+global yearlist 2023(1)2023
+global wavelist 5(1)1
 *global wavelist 1(1)4
 
 do ${extraction_code}/copy_over_raw_mrip.do
+
+/* copying over full years of data */
+global yearlist 2019(1)2022
+global wavelist 1(1)6
+
+do ${extraction_code}/copy_over_raw_mrip.do

stata_code/data_extraction_processing/processing/batch_file_to_process_monthly_mrip_data.do

@@ -65,7 +65,7 @@ save "${my_outputdir}/atlanticcod_landings_2b95.dta", replace
 /* loop over calendar years */
 /********************************************************************************/
 /********************************************************************************/
-foreach yr of numlist 2021 2022{
+foreach yr of global process_list {
 	global working_year `yr'
 	global previous_year=$working_year-1
 
@@ -281,7 +281,7 @@ foreach sp in atlanticcod haddock{
 
 
 
-/*stack these into a single dataset */
+/*stack the weighted B2s into a single dataset */
 clear
 local cod_wave_b2: dir "$my_outputdir" files "atlanticcodl_in_bin_b2*.dta"
 local haddock_wave_b2: dir "$my_outputdir" files "haddockl_in_bin_b2*.dta"
@@ -309,8 +309,6 @@ keep if year==$working_year
 save "$my_outputdir/haddock_b2_$working_year.dta", replace
 
 
-
-
 /* join the b2 length data with the total number of released to get the length distribution for the number of fish released */
 do "${processing_code}/monthly/process_b2_cod.do"
 do "${processing_code}/monthly/process_b2_haddock.do"
@@ -330,6 +328,16 @@ do "${processing_code}/monthly/haddock_plus_directed_trips_by_month.do"
 
 do "${processing_code}/monthly/cod_haddock_directed_trips_by_month.do"
 
+
+
+do "${processing_code}/mode/cod_haddock_directed_trips_by_month_mode.do"
+
+
+
+do "${processing_code}/mode/cod_haddock_directed_trips_by_month_disagg_mode.do"
+
+
+
 /* caught/targeted cod by wave */
 do "${processing_code}/monthly/cod_directed_trips_by_month.do"
 

stata_code/data_extraction_processing/processing/batch_lengths_by_mode.do

@@ -0,0 +1,260 @@
+/* Compute lengths by grouped_mode
+
+a, b1, b2 by for (ForHire=Party+charter and Private=Private+shore)
+
+*/
+
+
+global workdir "${data_main}/MRIP_$vintage_string"
+capture mkdir "$workdir"
+global my_outputdir "${workdir}/mode"
+capture mkdir "$my_outputdir"
+
+
+
+
+
+
+/*  */
+
+
+/*Set up the catchlist, triplist, and b2list global macros. These hold the filenames that are needed to figure out the catch, length-frequency, trips, and other things.*/
+
+
+
+
+/********************************************************************************/
+/********************************************************************************/
+/* loop over calendar years */
+/********************************************************************************/
+/********************************************************************************/
+foreach yr of global process_list {
+	global working_year `yr'
+	global previous_year=$working_year-1
+
+global wavelist 1 2 3 4 5 6
+global species1 "atlanticcod"
+global species2 "haddock"
+/*this is dumb, but I'm too lazy to replace everything that referred to these local/globals */
+
+
+
+/*catchlist -- this assembles then names of files that are needed in the catchlist */
+/*Check to see if the file exists */	/* If the file exists, add the filename to the list if there are observations */
+
+
+global catchlist: dir "${data_raw}" files "catch_$working_year*.dta" 
+global triplist: dir "${data_raw}" files "trip_$working_year*.dta" 
+global b2list: dir "${data_raw}" files "size_b2_$working_year*.dta" 
+global sizelist: dir "${data_raw}" files "size_$working_year*.dta" 
+
+
+/* catch frequencies per trip*/
+
+/* need to drop 
+id_code=="1757420210428005" from the sizelist, triplist, and catchlist. This is a trip that took place inshore in 2020 that caught 2 fish that were probably not cod
+
+
+id_code="1792820210410001" should be dropped from the triplist catchlist when computing targeted trips. 
+
+no changes to triplist or b2list
+
+The syntax will be to set up a global and then do 
+cap drop $drop_conditional after loading in the catchlist or sizelist files
+
+		If we don't need to drop anything, the capture will swallow the error term.
+		And if we do, the code will drop the appropriate rows.
+*/
+
+global drop_conditional 
+
+
+
+foreach sp in atlanticcod haddock{
+	global my_common `sp'
+
+	if "$my_common"=="atlanticcod"{
+	   global drop_conditional `"if id_code=="1757420210428005"  "'
+	}
+	else {
+    global drop_conditional 
+	}
+
+	do "${processing_code}/mode/domain_catch_mode_frequencies_gom_month.do"
+}
+
+/* catch totals  -- these are done for all 3 years at once*/
+
+global my_common "atlanticcod"
+	if "$my_common"=="atlanticcod"{
+	   global drop_conditional `"if  id_code=="1757420210428005"  "'
+	}
+	else {
+		global drop_conditional 
+	}
+
+do "${processing_code}/mode/domain_cod_mode_monthly_catch_totals.do"
+
+
+
+
+/* this is the place to "hack" using the trimmed landings from scott 
+
+read it in and merge.  rename landings old_landings. and rename trim_landings just landings
+*/
+
+
+
+
+use "$my_outputdir/atlanticcod_catch_mode_$working_year.dta", clear
+keep year month mode tot_cat claim harvest release
+rename claim a
+rename harvest b1
+rename release b2
+rename tot_cat tot_catch
+gen landings=a+b1
+capture destring month, replace
+
+
+save "$my_outputdir/atlanticcod_landings_mode_$working_year.dta", replace
+
+
+
+global my_common "haddock"
+
+	if "$my_common"=="atlanticcod"{
+   global drop_conditional `"if  id_code=="1757420210428005"  "'
+}
+else {
+    global drop_conditional 
+	}
+
+
+
+do "${processing_code}/mode/domain_haddock_mode_monthly_catch_totals.do"
+
+
+use "$my_outputdir/haddock_catch_mode_$working_year.dta", clear
+keep year month mode tot_cat claim harvest release
+rename claim a
+rename harvest b1
+rename release b2
+rename tot_cat tot_catch
+gen landings=a+b1
+capture destring month, replace
+save "$my_outputdir/haddock_landings_mode_$working_year.dta", replace
+
+
+
+
+
+
+
+
+clear
+
+/* length frequencies */
+
+
+
+foreach sp in atlanticcod haddock{
+	global my_common `sp'
+
+	if "$my_common"=="atlanticcod"{
+	   global drop_conditional `"if  id_code=="1757420210428005"  "'
+}
+	else {
+    global drop_conditional 
+	}
+
+	do "${processing_code}/mode/length_freqs_by_mode_month_gom.do"
+}
+
+
+/*stack together multiple, cleanup extras and delete */
+local cod_wave_ab1: dir "$my_outputdir" files "atlanticcodl_in_bin_mode_a1b1*.dta"
+
+clear
+foreach file of local cod_wave_ab1{
+	append using ${my_outputdir}/`file'
+	! rm ${my_outputdir}/`file'
+}
+keep if year==$working_year
+save "$my_outputdir/cod_ab1_mode_$working_year.dta", replace
+
+
+
+
+
+/*stack together multiple, cleanup extras and delete */
+local haddock_wave_ab1: dir "$my_outputdir" files "haddockl_in_bin_mode_a1b1*.dta"
+clear
+foreach file of local haddock_wave_ab1{
+	append using ${my_outputdir}/`file'
+	! rm ${my_outputdir}/`file'
+}
+keep if year==$working_year
+
+save "$my_outputdir/haddock_ab1_mode_$working_year.dta", replace
+
+
+
+
+
+
+/* B2 length frequencies per wave*/
+foreach sp in atlanticcod haddock{
+	global my_common `sp'
+	if "$my_common"=="atlanticcod"{
+	   global drop_conditional `"if  id_code=="1757420210428005"  "'
+}
+	else {
+    global drop_conditional 
+	}
+
+	do "${processing_code}/mode/b2_length_freqs_by_mode_month_gom.do"
+}
+
+
+
+
+/*stack the weighted B2s into a single dataset */
+clear
+local cod_wave_b2: dir "$my_outputdir" files "atlanticcodl_in_bin_mode_b2*.dta"
+local haddock_wave_b2: dir "$my_outputdir" files "haddockl_in_bin_mode_b2*.dta"
+
+	clear
+
+foreach file of local cod_wave_b2{
+	append using ${my_outputdir}/`file'
+	! rm ${my_outputdir}/`file'
+}
+keep if year==$working_year
+
+capture destring month, replace
+save "$my_outputdir/cod_b2_mode_$working_year.dta", replace
+	clear
+
+foreach file of local haddock_wave_b2{
+	append using ${my_outputdir}/`file'
+	! rm ${my_outputdir}/`file'
+}
+
+capture destring month, replace
+keep if year==$working_year
+
+save "$my_outputdir/haddock_b2_mode_$working_year.dta", replace
+
+
+/* this should work up to here */
+
+
+/* join the b2 length data with the total number of released to get the length distribution for the number of fish released */
+do "${processing_code}/mode/process_b2_mode_cod.do"
+do "${processing_code}/mode/process_b2_mode_haddock.do"
+
+}
+
+
+
+