From d73d2f052cba71cc1b74097a2faf4900cce47247 Mon Sep 17 00:00:00 2001 From: Rebecca Cremona Date: Sat, 9 Dec 2023 19:39:35 -0500 Subject: [PATCH] Remove LOCKSS, Part I. --- .../cc/perma/plugin/PermaCrawlSeed.java | 101 -------------- .../perma/plugin/PermaCrawlSeedFactory.java | 13 -- .../cc/perma/plugin/PermaPlugin.xml | 73 ---------- perma_web/conftest.py | 3 +- perma_web/fixtures/mirrors.json | 14 -- perma_web/lockss/README.md | 87 ------------ perma_web/lockss/admin.py | 12 -- .../lockss/migrations/0003_delete_mirror.py | 16 +++ perma_web/lockss/models.py | 46 ------- .../static/lockss/plugins/PermaPlugin.jar | Bin 5790 -> 0 bytes .../static/lockss/plugins/plugin.keystore | Bin 910 -> 0 bytes .../lockss/static/lockss/plugins/plugins.html | 12 -- .../templates/lockss/daemon_settings.txt | 28 ---- perma_web/lockss/templates/lockss/titledb.xml | 80 ----------- perma_web/lockss/tests/__init__.py | 0 perma_web/lockss/tests/test_views.py | 39 ------ perma_web/lockss/urls.py | 13 -- perma_web/lockss/views.py | 127 ------------------ .../settings/deployments/settings_common.py | 2 +- perma_web/tasks/dev.py | 2 +- perma_web/urls.py | 1 - 21 files changed, 19 insertions(+), 650 deletions(-) delete mode 100644 extras/lockss-plugin/cc/perma/plugin/PermaCrawlSeed.java delete mode 100644 extras/lockss-plugin/cc/perma/plugin/PermaCrawlSeedFactory.java delete mode 100644 extras/lockss-plugin/cc/perma/plugin/PermaPlugin.xml delete mode 100644 perma_web/fixtures/mirrors.json delete mode 100644 perma_web/lockss/README.md delete mode 100644 perma_web/lockss/admin.py create mode 100644 perma_web/lockss/migrations/0003_delete_mirror.py delete mode 100755 perma_web/lockss/models.py delete mode 100644 perma_web/lockss/static/lockss/plugins/PermaPlugin.jar delete mode 100644 perma_web/lockss/static/lockss/plugins/plugin.keystore delete mode 100644 perma_web/lockss/static/lockss/plugins/plugins.html delete mode 100644 perma_web/lockss/templates/lockss/daemon_settings.txt delete mode 100644 perma_web/lockss/templates/lockss/titledb.xml delete mode 100644 perma_web/lockss/tests/__init__.py delete mode 100644 perma_web/lockss/tests/test_views.py delete mode 100755 perma_web/lockss/urls.py delete mode 100755 perma_web/lockss/views.py diff --git a/extras/lockss-plugin/cc/perma/plugin/PermaCrawlSeed.java b/extras/lockss-plugin/cc/perma/plugin/PermaCrawlSeed.java deleted file mode 100644 index 3917e9e82..000000000 --- a/extras/lockss-plugin/cc/perma/plugin/PermaCrawlSeed.java +++ /dev/null @@ -1,101 +0,0 @@ -package cc.perma.plugin; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.lang.Integer; -import java.net.URL; -import java.util.ArrayList; -import java.util.Collection; - -import org.lockss.config.Configuration; -import org.lockss.crawler.BaseCrawlSeed; -import org.lockss.daemon.ConfigParamDescr; -import org.lockss.daemon.Crawler.CrawlerFacade; -import org.lockss.daemon.PluginException; -import org.lockss.plugin.ArchivalUnit; -import org.lockss.plugin.ArchivalUnit.ConfigurationException; -import org.lockss.plugin.AuUtil; - - -public class PermaCrawlSeed extends BaseCrawlSeed { - - protected Collection urls; - protected String baseUrl; - protected Integer year; - protected Integer month; - - public PermaCrawlSeed(CrawlerFacade cf) { - super(cf); - } - - @Override - protected void initialize() - throws PluginException, ConfigurationException, IOException { - super.initialize(); - - // get configuration - Configuration config = au.getConfiguration(); - this.baseUrl = config.get(ConfigParamDescr.BASE_URL.getKey()); - try{ - this.year = config.getInt(ConfigParamDescr.YEAR.getKey()); - } catch (Configuration.InvalidParam e) { - throw new ConfigurationException("Year must be an integer", e); - } - try{ - this.month = config.getInt("month"); - } catch (Configuration.InvalidParam e) { - throw new ConfigurationException("Month must be an integer", e); - } - } - - @Override - public Collection doGetStartUrls() - throws ConfigurationException, PluginException, IOException{ - - if(urls == null){ - urls = new ArrayList(); - Integer offset = 0; - long startDate = AuUtil.getAuState(au).getLastCrawlTime(); - if(startDate>0) - startDate = startDate/1000 - 24*60*60; // convert to seconds from milliseconds, and subtract one day - - while(true){ - URL url = new URL(String.format("%slockss/search/?offset=%s&creation_month=%s&creation_year=%s&updates_since=%s", baseUrl, offset, month, year, startDate)); - InputStream is = url.openStream(); - BufferedReader reader = new BufferedReader(new InputStreamReader(is)); - String targetUrl; - - final ArrayList partial = new ArrayList(); - try { - while ((targetUrl = reader.readLine()) != null) { - partial.add(String.format("%slockss/fetch/%s", baseUrl, targetUrl)); - } - } finally { - is.close(); - } - if(partial.size()>0){ - urls.addAll(partial); - offset += 1000; - }else{ - break; - } - } - } - - return urls; - } - - - /** - * All URLs are start urls so don't fail on error - * @return false - */ - @Override - public boolean isFailOnStartUrlError() { - return false; - } - - -} diff --git a/extras/lockss-plugin/cc/perma/plugin/PermaCrawlSeedFactory.java b/extras/lockss-plugin/cc/perma/plugin/PermaCrawlSeedFactory.java deleted file mode 100644 index 04504493c..000000000 --- a/extras/lockss-plugin/cc/perma/plugin/PermaCrawlSeedFactory.java +++ /dev/null @@ -1,13 +0,0 @@ -package cc.perma.plugin; - -import org.lockss.crawler.CrawlSeed; -import org.lockss.crawler.CrawlSeedFactory; -import org.lockss.daemon.Crawler; - -public class PermaCrawlSeedFactory implements CrawlSeedFactory { - @Override - public CrawlSeed createCrawlSeed(Crawler.CrawlerFacade crawlFacade) { - return new PermaCrawlSeed(crawlFacade); - } - -} diff --git a/extras/lockss-plugin/cc/perma/plugin/PermaPlugin.xml b/extras/lockss-plugin/cc/perma/plugin/PermaPlugin.xml deleted file mode 100644 index 3f2765a46..000000000 --- a/extras/lockss-plugin/cc/perma/plugin/PermaPlugin.xml +++ /dev/null @@ -1,73 +0,0 @@ - - - plugin_config_props - - - base_url - Base URL - Usually of the form http://<journal-name>.com/ - 3 - 40 - true - false - - - year - Year - Four digit year (e.g., 2004) - 4 - 4 - true - false - - - month - Month - Archive creation month (between 1 and 12) - 2 - 2 - true - false - - - - - au_name - "Perma Captures For %d-%.02d", year, month - - - au_permission_url - "%slockss/permission/", base_url - - - au_def_pause_time - 1000 - - - au_def_new_content_crawl - 300000 - - - plugin_name - Perma Plugin - - - plugin_notes - notes - - - plugin_identifier - cc.perma.plugin.PermaPlugin - - - au_crawlrules - - 4,"^%s", base_url - 1,"^%slockss/fetch/.*\.warc\.gz", base_url - - - - plugin_crawl_seed_factory - cc.perma.plugin.PermaCrawlSeedFactory - - \ No newline at end of file diff --git a/perma_web/conftest.py b/perma_web/conftest.py index 0591d11cd..32aab89a4 100644 --- a/perma_web/conftest.py +++ b/perma_web/conftest.py @@ -77,8 +77,7 @@ def _load_json_fixtures(): 'fixtures/users.json', 'fixtures/api_keys.json', 'fixtures/folders.json', - 'fixtures/archive.json', - 'fixtures/mirrors.json' + 'fixtures/archive.json' ]) diff --git a/perma_web/fixtures/mirrors.json b/perma_web/fixtures/mirrors.json deleted file mode 100644 index b653d0e17..000000000 --- a/perma_web/fixtures/mirrors.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "pk": 1, - "model": "lockss.mirror", - "fields": { - "name": "Test Mirror", - "ip": "203.0.113.3", - "hostname": "mirror.example.com", - "peer_port": 9729, - "content_url": "https://mirror.example.com:8080/", - "enabled": true - } - } -] diff --git a/perma_web/lockss/README.md b/perma_web/lockss/README.md deleted file mode 100644 index 51a1c42ad..000000000 --- a/perma_web/lockss/README.md +++ /dev/null @@ -1,87 +0,0 @@ -# LOCKSS Installation Instructions - -## Add LOCKSS repo: - - $ sudo yum install yum-utils - $ sudo yum-config-manager --add-repo http://www.lockss.org/repo/ - $ sudo rpm --import http://www.lockss.org/LOCKSS-GPG-RPM-KEY - -## Install packages: - - $ sudo yum install -y lockss-daemon java-1.8.0-openjdk-headless.x86_64 ntp - -## Enable services: - - $ sudo chkconfig --add lockss - $ sudo chkconfig lockss on - $ sudo chkconfig ntpd on - -## Make LOCKSS storage dir: - - $ sudo mkdir -p /srv/caches/cache0 - -## Lock down SSH by subnet: - -(If not handled by router/firewall.) - -## Open LOCKSS ports: - - $ sudo iptables -I INPUT 1 -m state --state NEW -m tcp -p tcp --dport 8081 -j ACCEPT - $ sudo iptables -I INPUT 1 -m state --state NEW -m tcp -p tcp --dport 8082 -j ACCEPT - $ sudo iptables -I INPUT 1 -m state --state NEW -m tcp -p tcp --dport 9729 -j ACCEPT - -## Configure LOCKSS daemon: - - $ sudo /etc/lockss/hostconfig - root is configuring - eth0: error fetching interface information: Device not found - LOCKSS host configuration for Linux. - For more information see /etc/lockss/README - Configuring for user lockss - Fully qualified hostname (FQDN) of this machine: [hlslpermamirror.law.harvard.edu] - IP address of this machine: [] - Is this machine behind NAT?: [N] - Initial subnet for admin UI access: [140.247.203.0/24] - LCAP V3 protocol port: [9729] - PROXY port: [8080] - Admin UI port: [8081] - Mail relay for this machine: [localhost] - Does mail relay localhost need user & password: [N] - E-mail address for administrator: [] - Path to java: [/usr/bin/java] - Java switches: [] - Configuration URL: [http://props.lockss.org:8001/daemon/lockss.xml] https://perma.cc/lockss/daemon_settings.txt - Verify configuration server authenticity?: [Y] - Server certificate keystore: [lockss-ca] /etc/lockss/perma-test.keystore - Configuration proxy (host:port): [NONE] - Preservation group(s): [prod] perma - Content storage directories: [] /srv/caches/cache0 - Temporary storage directory: [/srv/caches/cache0/tmp] - User name for web UI administration: [] lockss - Password for web UI administration user lockss: [] - Password for web UI administration (again): [] - - Configuration: - ... - OK to store this configuration: [Y] - -## Import Perma server certificate: - - sudo keytool -importkeystore -srckeystore /etc/lockss/perma-test.keystore -destkeystore /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.60-2.b27.el7_1.x86_64/jre/lib/security/cacerts - -## Start daemon: - - sudo service lockss start - -## Subscribe to Perma Archival Units: - -- Go to Journal Configuration. -- Click Add AUs. - -(TODO: remove necessity to do this?) - -## Further reading: - -- http://plnwiki.lockss.org/wiki/index.php/LOCKSS_Technical_Manual#How_to_install_and_configure_a_locks_daemon_on_a_PLN_node.3F -- http://webcache.googleusercontent.com/search?q=cache:EdUEEhW-jsYJ:metaarchive.org/public/ddp/chapters/Hea-Fen-Pit-2_2.doc+&cd=4&hl=en&ct=clnk&gl=us -- http://www.lockss.org/docs/LOCKSS-Linux-Install.pdf \ No newline at end of file diff --git a/perma_web/lockss/admin.py b/perma_web/lockss/admin.py deleted file mode 100644 index e0ff9e0f7..000000000 --- a/perma_web/lockss/admin.py +++ /dev/null @@ -1,12 +0,0 @@ -from django.contrib import admin - -from .models import Mirror - - -class MirrorAdmin(admin.ModelAdmin): - - list_display = ['name', 'ip', 'hostname', 'content_url', 'enabled',] - list_editable = ['ip', 'hostname', 'content_url', 'enabled',] - - -admin.site.register(Mirror, MirrorAdmin) diff --git a/perma_web/lockss/migrations/0003_delete_mirror.py b/perma_web/lockss/migrations/0003_delete_mirror.py new file mode 100644 index 000000000..08966c7c7 --- /dev/null +++ b/perma_web/lockss/migrations/0003_delete_mirror.py @@ -0,0 +1,16 @@ +# Generated by Django 4.2.8 on 2023-12-10 00:14 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('lockss', '0002_auto_20180625_2258'), + ] + + operations = [ + migrations.DeleteModel( + name='Mirror', + ), + ] diff --git a/perma_web/lockss/models.py b/perma_web/lockss/models.py deleted file mode 100755 index 0c768cce0..000000000 --- a/perma_web/lockss/models.py +++ /dev/null @@ -1,46 +0,0 @@ -from django.db import models -from django.core.cache import cache as django_cache - - -class Mirror(models.Model): - name = models.CharField(max_length=255) - ip = models.CharField(max_length=15, help_text="E.g. 12.34.56.78") - hostname = models.CharField(max_length=255, help_text="E.g. mirror.example.com") - peer_port = models.IntegerField(default=9729, help_text="Port where this mirror listens for other LOCKSS nodes.") - content_url = models.URLField(max_length=255, help_text="E.g. https://mirror.example.edu:8080/") - enabled = models.BooleanField(default=True) - - class Meta: - ordering = ['name'] - - def __unicode__(self): - return self.name - - def save(self, *args, **kwargs): - super(Mirror, self).save(*args, **kwargs) - self._invalidate_cached_mirrors() - - def delete(self, *args, **kwargs): - super(Mirror, self).delete(*args, **kwargs) - self._invalidate_cached_mirrors() - - @classmethod - def get_cached_mirrors(cls): - """ - Get a list of {'name':,'content_url':,'ip':} dicts for each active mirror. - Keep result in cache. - """ - # get mirror list from cache - mirrors = django_cache.get('mirrors') - attrs = ('name', 'content_url', 'ip') - if mirrors is None: - mirrors = [dict((attr, getattr(mirror, attr)) for attr in attrs) for mirror in cls.objects.filter(enabled=True)] - django_cache.set('mirrors', mirrors) - - # return mirrors - if mirrors: - return mirrors - return [] - - def _invalidate_cached_mirrors(self): - django_cache.delete('mirrors') diff --git a/perma_web/lockss/static/lockss/plugins/PermaPlugin.jar b/perma_web/lockss/static/lockss/plugins/PermaPlugin.jar deleted file mode 100644 index 58508809a51ba5f3c884ed79c262f59a05a0c257..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5790 zcmai21yogAw?33~Hz*uZ8UzmQLAv3319^6XXr)UlgQ{k*l zVpJNPySzAipl8OMP~-Ye+`PE*i3-1ZXX!@`o2Xt?6#Wuw<%9QyKrsg;LfZ-ON_WF9 zc+0YT4o}1bG2S1s1K9!*K6_bJm>(9dV1*zK%**VEAst{YdUquo^C`(##j>&^+dOkz z?q!a8!o}B#Q#xpxMc$SuX2_S~)SY9J%IZk0c@Si2aM?Cuh}42;6=@UsIfhJ^G05YD z^CS%=%UWcU(D&zx$UJ?NQL3r8L797Pi^oeXD*l)KZKiR+GtR!Nhy!b#bJH*V-|E^2 zt6pTF44bat&?alG7in(}P7myj0DYd;?DDF(J*YCHPf|M*g1yw`-8Y&?fGD7%(Z1M{ zV?Y1^+O8kSziu=B-ED?QsVRwZK&4+r_g(KXK!pO*pq%31OSbP)=TuYPqc(BvncN13syY4kK}hKKxgl>JoN*APGg+TE_EUfbm223l2bmpiiK5hgjkp~mf%d$-8{V;ifHRr6+ zfGN<%5St!U6-4K-TThJ>ghZ{;@tRAMY>`+^d&;a@a{|8f+?kC|d_=GYYuaqAK^= zQM$6@>sPOmG!n7J5j>v?@E<`M6b0>>Uk8W{@&5}F!T$}CBvgzCW*lm`JZAL^b~f18 zGg$pDv~%?ONeGP{j+yq0kJLzuPLT^sX!WCfAL1ZJ=;S9xScQw{#BOMAY-((*dh*A>lL1{bDF#gd~O+ERozN_~;=IdkYgM&oHFt34p$bqn|t` zIlgQm85(ELWCBDXSSBFB;gIV`Vo_8@8|Y(GU}qU*E8zXczkt%~h*@=VEbN7fN{K!7gE*|?^hX=M2Hd_QXHoGELq+(WwN2H$~9VU|-K!2j+!-EhP!`2S8 zZ#_I9KJN1pT)Wro#3NV@+aQ_kf%w&VMlFM@x{Ks}+#fru-I{EK z-z`pN&wuWz*0(F>(zK}>6Gnba@G;1HDpSKhN4xO5o7*vnbNv(6C)Mtrf{kZMiOgF2x(UX0Z{(YfH{Q$)mjbLm zTA-uYc&V(aq*VKUx4wk_a#bWm%lJ^JpzPhg9`U)U7olYHitKBk8XxCodvS7Vi?dKh#8@x115Pyg71)*^>X|) zH*a3c_rz|a(lYpxq>7WNH$x|*^+~W?$7=BIk5SfN-pBma+^=UEY1Q`ZP&nh!>h>5X zQbO|_*~soUn?w(iB(D}fhnMeWiY*Ho*7Cq?$UYEZ4D>LaP$g>XkZbQh@d-QLa5{;8 zz4KXgf?>6g+Q1?;&wTSUeP+|#_puTc5&l7$VoxEsY07wDT?y57N<>BDq!PjdBLM(< z$ba5G3;+NP09gI45}^Z-Z!6K|=ewq9wmH(*ck@5E_60C8;k-$=fBO2}0R2yc=Vp$! zMt98g;`9FI2k)-ga~l^6OS?a;|16biE|h;OmGu8)ee(wiM0<(oP3|=t&6i!+eC1n(9RUScZy5`6Y3I zX-KK~`x!jDepMKhps&DELQt3KwebD#r0!Hcu&aAiT-(L;GGTDJa_XT?Z@722@O(tZ z)pooJk~{Y0RyW7DjvZI(Hm(iBwmq8Ycpsd+bAd>Y)(dePPS`AEmh3?ryiflfcy^D8u={6>k8m|pB{)x z4#*@U6%y|(P2OwDYr<5Ln%f2N^dByc#mK7)CCnvPE~HQjRA70aQFx&rTz>)4B%8t; z=lQDbW2ei5Q!r|&B#@?*zusQ>KHMdAt=8OSYnV!jzD;l-8QNa&49>A=BQ!R88f~c? z_ZCiXPVYQh%}Q`iE3U8f?jTz6GRtgWoN8RA%EP75yEVrpNaM$ zVUPC*GltxxC!G=oyqgW=V5~kcwfME?m~U5pHAv-Qb$Sf9nYT=r8v`$W)fp%pBM4+h zmj{}xp>oypP`wPI70%LGEa=VeiS7BGORF(dxmhoQO`$vNFen$<1Jfs-jL%3J#$p$= zbV6pV_mZ!&dEP7N*>+x{TRj^>79>PHuB4#dKiHp~Dg}bNH*3`-KU??vh#^w zO;E=%us&jYSNpU37}yv0^YOTUCYxVcqzZG3V7OK?niZ98cT^_=?^lxD~rYx0(HTo!+j23lBR(QNRDz7OE>v30Yi%Q#cC#anPnAqr8 z+bT_d)BxsKf|!?y$0Dsc;^ljb){9t_c7cf~wj=L#=)4LiF|Ed$VCQfdp`qu~2MizR zgeO2R=SVgiLbsX*XliOhUSJ$qnft4H9!VOvciWJ;3Cx1wOd++hzFQM347^Ec8V4vp z+%L(3+9HEb>?4%>Dc8j!hR5IEAXgOOvD?MG|wr zpduGG>H7?E7AnNpzCiaZ3eE90BHhI}SKk9ZjV5EGtU}8v0nzLs;%$dHPHe3gQRcRR zD7fmUvq3*VJr9xL(Zs7Gi$#N&=5pSU;*CnO$n6SP!^%o{e{}+2$x1 zC5WlkZnRbNW1-vb?B)t*8H%iT3lhkk9o~dsi&G}zwZ%RV_d!N*u5%n{W-Z_)K8qr| zYCFsCxIaC{H)f1Ud;V4LbsH7d)V4G3p>Z9PnFcB5w>-4C&c-3 zxVAwW!1U^?U*id-S$>vYOd-kYDaIB7>jtd^4Wff~SLt$_83;c@R zfwZ%!;zOe_N!fP`6Sl3z?2aYJGlH)s3X{5@YQb)wnnR{ft&)cEpGjMYK%Ra&M0DNv zkWV}GKvI0cHTO_7yad7CM9Vg3*N+X0oz~gZ=zAw1e4^R!6r6EEyrYlfPCD#_V@n(3 zcSK-U)*8DdwK=oqxo&{p+xA(k)iWJkq>ohhr5Nck8S7*|p;@XVPq#sjq#@nua9f*A z`x~}NU^t;zoSbNDbp}o0U?mZ56)gJE32E{<;;IE#*J_7AsUhN(Y1q{(I#=?CDNFP< zI8HxvU&XUMPoI1&vGG_yz8c(-+4O2vEO$1`opnw+lBCJcM1-)(^Vpz)v{3&zN#Rw- zN`|W{IzejfXlC8YWZ=&3EjUl%wifY)|8|CAV z0y{!+ll?5Lx_ezztZz;T_xU30M|!sw+0Yw)439&9RxlU7;mPS?mwFly@z`7kdx5xm zCfeTU)ABOD@Bqv4hxpK=)+5oIc89$kXYCvn0ARdsFK!j#@4W}>f9)oujZB>F9X1Z0?zV1?L3o&E=Zl?+4Fxg0FpLEM=;@zo&{0Xgf~9BcP_b6rovaQy zitw<+Q*H~3m40$_*SWcjX9)M-8PbLdBK=}C*5-=gFNYb1(zw6`6Mg&f<~b(F2+Zo< zlXbBsW(^AcY#2$(w1b)!F3B6g18WgQF&NLS`%ESaWqa)t>@a<--BJ_VIT7=UJDc@2 z3LR*(w>N?b*5ykA%TCFWSh2F+7x4~Dei8B75Rgx@UGxYWQ>kMuwR4%+5=fgX?%876_rldMsfl_ON)U;nFRux_5fjTRG$4vRH_pP*cm zoG^n^5u3nNIz^va8cUv!WXU1KQX{;k8%H0DV?lcES<6E7WSVLG z@x)t^{w5DBT#*F}Ufen0w*am}=fs*CRmE(v!SL6iTyeBUmdUIs))~AN6g???HI}Gq zImvx%s@C_RJ=vf7Se%rI2f#0*)G=!oWS9a_3u)MhjzTG2Dw5aJNB39$Yg=a) z8T-wTi&%g*aBsn zsKmCD&z{60Zrsn$l!oIP)&dZf@QEL#4l4JF2y^_}q>h_#{4iv0TDXXWD@?cCh1lh6CMi9+zapo8|9yHO6 z4LsgGJshK~Dm6rsgn8t=ya>-DzaPk9Vphlp{yb13eigvD4_)l*7`{y*_GL8h+Js_-H|s$jVPZ z5oVgrf-3vLxmVO?YlF*$zgcA_{u}wt*$my@I8eEsZX{nTD~JLDA`#$r-gkFIbu;h# zJN{|<_q&I?^OKuV)-5cq@56sQ{59D6oqksU-V9l8fpvYAexqNHTyFRnfPa^XxBRes zcldvb##`#|s_;fO-lFE3`iF@8cfyT~yv4+ygug`PE$#Qzej_VyadDUSACY+*p_}}1 zw`APN%v-Qt7n18${`u3t*#96p?{4|cdhX_`n^JQN58VIe@o$CauH&1f-aTkHsqPkP f*C*p2mwKD{APOkgy8*ZkEe0U<`m}Bl-hBNJOf9TX diff --git a/perma_web/lockss/static/lockss/plugins/plugin.keystore b/perma_web/lockss/static/lockss/plugins/plugin.keystore deleted file mode 100644 index 9587b4c93aa5e98aaf6db360e304c6bc32893471..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 910 zcmezO_TO6u1_mY|W(3o$1*t{3i9nuztH8Qi2G$5YQv*u|2Ic^RCT1TX=32na#K^?N zqVnPRUIT76cC9v#a~3SjtPBPfhTI06Y|No7Y{E>T!G^*Hf*=luFqd;;PG(wuQC?=E zp^$+9NRVBaGaxZ1U%@e_B;QcRKnf(rB`h3Vl9&fnn4%DxmsyrtRGe8-X~=KD3zFp) zX7llP_6`m<6g1!m2`~$D1b|$smz-=MC(dhVYG7e(XlP+#Xk;2C&T9nZ8d*TO1L(3Q z#_a}8j5@GjVgz#JnHn1z{?-RK%9aXg2Hp3Xqxa^?^KXLNgKXZrFdUY%Z`c;7mgySn zpcME!(zZ9^*Z!klcNu&;Jv%h#dZTrNtV#Z-w1n;1GXs_`nLqypOSOQ~A^+Vgum9VA zO!|j)`14~cmWlGS{BHhs`Ka7sCesk%D6xq~-%3BP(@wmu_5PC*v*g^19*oDCL>ZQ_-Kd;Z$Hmz)N^9lLmRQG?vR>ZjR%X@$!Tw1 zRyr|naXt4hMbo5@zmuvMRjkh}`M7UI;DpW<&G#QhJ)O@ssXHyMp*mDGh0(|Q-+50h z=EfEVrp5;M&($k8>^rUy^E}dHn_kC8qf6c&{;vELAX+@VoZbCP%akj6txB=mP9|ut1*SAvJ{B<+ z5pk!Zfz|q^r{7n9`EKg6X@~w6vo9#gpIM*5K!-`h=ZEp@(<}3Q_&2mNl-#=_Qu1yE xFOvwj@+Q`qiKlWb7#zK0kIojD=d5sdlGKD*qB#pIwe7mvPTfDG - - Perma Plugin Registry - - -

Perma Plugin Registry

- -

LOCKSS system has permission to collect, preserve, and serve this Archival Unit

- - diff --git a/perma_web/lockss/templates/lockss/daemon_settings.txt b/perma_web/lockss/templates/lockss/daemon_settings.txt deleted file mode 100644 index ea31595e7..000000000 --- a/perma_web/lockss/templates/lockss/daemon_settings.txt +++ /dev/null @@ -1,28 +0,0 @@ -# peer list -{% if mirrors %} -org.lockss.id.initialV3PeerList=\{% for mirror in mirrors %} - TCP:[{{ mirror.ip }}]:{{ mirror.peer_port }}{% if not forloop.last %};\{% endif %}{% endfor %} -{% endif %} - -# plugins -org.lockss.plugin.registries={{ static_url_prefix }}lockss/plugins/plugins.html -org.lockss.plugin.registries.crawlInterval={{ crawl_interval }} -org.lockss.plugin.keystore.location={{ static_url_prefix }}lockss/plugins/plugin.keystore - -# titledbs -org.lockss.titleDbs={{ django_url_prefix }}{% url 'lockss:titledb' %} - -# disable content proxy and put content server on port 8080 -org.lockss.contentui.port=8080 -org.lockss.contentui.start=true -org.lockss.icp.enabled=false -org.lockss.proxy.audit.start=false -org.lockss.proxy.port=8080 -org.lockss.proxy.start=false - -# allowed IP address ranges for content playback -org.lockss.proxy.access.ip.include={{ content_ips }} - -# set required quorum size -org.lockss.poll.quorum={{ quorum }} -org.lockss.poll.v3.quorum={{ quorum }} diff --git a/perma_web/lockss/templates/lockss/titledb.xml b/perma_web/lockss/templates/lockss/titledb.xml deleted file mode 100644 index 53c6d1676..000000000 --- a/perma_web/lockss/templates/lockss/titledb.xml +++ /dev/null @@ -1,80 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -]> - - - - - - - - - - - - - - - - - - {% for year, month in archival_units %} - - - - - - - - - - - - - - - - - - - - - - - {% endfor %} - - - - \ No newline at end of file diff --git a/perma_web/lockss/tests/__init__.py b/perma_web/lockss/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/perma_web/lockss/tests/test_views.py b/perma_web/lockss/tests/test_views.py deleted file mode 100644 index 81ef3283e..000000000 --- a/perma_web/lockss/tests/test_views.py +++ /dev/null @@ -1,39 +0,0 @@ -from django.test import Client -from io import BytesIO -from mock import patch -from perma.tests.utils import PermaTestCase - -class LockssTestCase(PermaTestCase): - - def setUp(self): - self.clients = [] - for (ip, status_code) in [('203.0.113.3', 200), ('203.0.113.4', 403)]: - self.clients.append((Client(REMOTE_ADDR=ip), status_code)) - - def test_daemon_settings(self): - for (client, status_code) in self.clients: - response = client.get('/lockss/daemon_settings.txt', secure=True) - self.assertEqual(response.status_code, status_code) - - def test_titledb_and_search(self): - response = self.clients[0][0].get('/lockss/titledb.xml', secure=True) - self.assertEqual(response.status_code, self.clients[0][1]) - for ym in [ym[0:7] for ym in str(response.content, 'utf-8').split('Perma.cc Captures For ')][1:]: - (year, month) = ym.split('-') - for (client, status_code) in self.clients: - response = client.get('/lockss/search/?creation_year={0}&creation_month={1}'.format(year, month), secure=True) - self.assertEqual(response.status_code, status_code) - response = self.clients[1][0].get('/lockss/titledb.xml', secure=True) - self.assertEqual(response.status_code, self.clients[1][1]) - - def test_permission(self): - for (client, status_code) in self.clients: - response = client.get('/lockss/permission/', secure=True) - self.assertEqual(response.status_code, status_code) - - @patch('lockss.views.default_storage.open', autospec=True) - def test_fetch(self, mock_open): - mock_open.return_value = BytesIO(b"warc placeholder") - for (client, status_code) in self.clients: - response = client.get('/lockss/fetch/3S/LN/JH/3SLN-JHX9.warc.gz', secure=True) - self.assertEqual(response.status_code, status_code) diff --git a/perma_web/lockss/urls.py b/perma_web/lockss/urls.py deleted file mode 100755 index dcb658aef..000000000 --- a/perma_web/lockss/urls.py +++ /dev/null @@ -1,13 +0,0 @@ -from django.urls import re_path -from . import views - -app_name='lockss' -urlpatterns = [ - re_path(r'^search/?$', views.search, name='search'), - re_path(r'^fetch/(?P(?:[A-Za-z0-9]{2}/)+)(?P.+)\.warc\.gz$', views.fetch_warc, name='fetch_warc'), - - # LOCKSS config URLs - re_path(r'^permission/$', views.permission, name='permission'), - re_path(r'^titledb.xml$', views.titledb, name='titledb'), - re_path(r'^daemon_settings.txt$', views.daemon_settings, name='daemon_settings'), -] diff --git a/perma_web/lockss/views.py b/perma_web/lockss/views.py deleted file mode 100755 index 258047b3d..000000000 --- a/perma_web/lockss/views.py +++ /dev/null @@ -1,127 +0,0 @@ -from urllib.parse import urljoin -from wsgiref.util import FileWrapper -from datetime import datetime, date -from dateutil.relativedelta import relativedelta -import logging - -from django.utils import timezone -from django.conf import settings -from django.core.files.storage import default_storage -from django.http import HttpResponse, HttpResponseBadRequest, Http404, StreamingHttpResponse, HttpResponseForbidden -from django.shortcuts import get_object_or_404, render - -from perma.models import Link -from perma.utils import get_client_ip -from .models import Mirror - -logger = logging.getLogger(__name__) - -### HELPERS ### - -def django_url_prefix(request): - http_prefix = "s" if settings.SECURE_SSL_REDIRECT else "" - return f"http{http_prefix}://{request.get_host()}" - -def allow_by_ip(view_func): - def authorize(request, *args, **kwargs): - user_ip = get_client_ip(request) - if settings.LOCKSS_DEBUG_IPS: - logger.warn("CACHED MIRRORS: {}".format(Mirror.get_cached_mirrors())) - logger.warn("CLIENT IP: {}".format(user_ip)) - logger.warn("request.META: {}\n".format(request.META)) - for mirror in Mirror.get_cached_mirrors(): - if user_ip == mirror['ip']: - return view_func(request, *args, **kwargs) - return HttpResponseForbidden() - return authorize - - -### VIEWS ### - -@allow_by_ip -def search(request): - updates = Link.objects.visible_to_lockss().order_by('archive_timestamp') - - # apply from_date - if 'updates_since' in request.GET: - try: - from_date = datetime.utcfromtimestamp(int(request.GET['updates_since'])) - except ValueError: - return HttpResponseBadRequest("updates_since must be an integer and a valid timestamp.") - updates = updates.filter(archive_timestamp__gte=from_date) - - # apply Archival Unit limits - try: - month = int(request.GET['creation_month']) - year = int(request.GET['creation_year']) - except (KeyError, ValueError): - return HttpResponseBadRequest("creation_month and creation_year must be integers.") - updates = updates.filter(creation_timestamp__year=year, creation_timestamp__month=month) - - # apply offset - try: - offset = int(request.GET.get('offset', 0)) - except ValueError: - return HttpResponseBadRequest("offset must be an integer.") - updates = updates[offset:offset+1000] - - # export file names - files_to_index = [] - for update in updates: - files_to_index.append(update.warc_storage_file().split(settings.WARC_STORAGE_DIR+'/')[1]) - - return HttpResponse("\n".join(files_to_index), content_type="text/plain") - - -@allow_by_ip -def fetch_warc(request, path, guid): - # fetch link and check path is correct - link = get_object_or_404(Link, pk=guid) - if path.rstrip('/') != link.guid_as_path() or link.archive_timestamp is None or link.archive_timestamp > timezone.now(): - raise Http404 - - # deliver warc file - with default_storage.open(link.warc_storage_file()) as file: - response = StreamingHttpResponse(FileWrapper(file, 1024 * 8), - content_type="application/gzip") - response['Content-Disposition'] = f"attachment; filename={link.guid}.warc.gz" - return response - - -@allow_by_ip -def permission(request): - return HttpResponse("LOCKSS system has permission to collect, preserve, and serve this open access Archival Unit") - - -@allow_by_ip -def titledb(request): - # build list of all year/month combos since we started, in the form [[2014, "01"],[2014, "02"],...] - first_archive_date = Link.objects.order_by('creation_timestamp')[0].creation_timestamp - start_month = date(year=first_archive_date.year, month=first_archive_date.month, day=1) - today = date.today() - archival_units = [] - while start_month <= today: - archival_units.append([start_month.year, f'{start_month.month:02}']) - start_month += relativedelta(months=1) - - return render(request, 'lockss/titledb.xml', { - 'archival_units': archival_units, - 'django_url_prefix': django_url_prefix(request), - }) - - -@allow_by_ip -def daemon_settings(request): - """ Generate settings files for our PLN nodes. """ - - url_prefix = django_url_prefix(request) - static_url_prefix = urljoin(url_prefix, settings.STATIC_URL) - - return render(request, 'lockss/daemon_settings.txt', { - 'django_url_prefix': url_prefix, - 'static_url_prefix': static_url_prefix, - 'mirrors': list(Mirror.objects.filter(enabled=True)), - 'content_ips': settings.LOCKSS_CONTENT_IPS, - 'quorum': settings.LOCKSS_QUORUM, - 'crawl_interval': settings.LOCKSS_CRAWL_INTERVAL, - }, content_type="text/plain") diff --git a/perma_web/perma/settings/deployments/settings_common.py b/perma_web/perma/settings/deployments/settings_common.py index 313349e93..8ed340ff1 100644 --- a/perma_web/perma/settings/deployments/settings_common.py +++ b/perma_web/perma/settings/deployments/settings_common.py @@ -387,7 +387,7 @@ # show info for our first-party apps **{ app_name: {'level': 'INFO'} - for app_name in ('api', 'lockss', 'perma',) + for app_name in ('api', 'perma',) }, } LOGGING['formatters'] = { diff --git a/perma_web/tasks/dev.py b/perma_web/tasks/dev.py index 89c865500..ec740d7cf 100644 --- a/perma_web/tasks/dev.py +++ b/perma_web/tasks/dev.py @@ -441,7 +441,7 @@ def check_storage(ctx, start_date=None): print("Building link cache ...") with open(link_cache, 'w') as tmp_file: capture_filter = (Q(role="primary") & Q(status="success")) | (Q(role="screenshot") & Q(status="success")) - # assemble list of links by year-month, as in lockss/views.titledb: + # assemble list of links by year-month start_month = date(year=start_datetime.year, month=start_datetime.month, day=1) today = date.today() while start_month <= today: diff --git a/perma_web/urls.py b/perma_web/urls.py index 2860b77eb..3f9a9e91e 100755 --- a/perma_web/urls.py +++ b/perma_web/urls.py @@ -9,6 +9,5 @@ urlpatterns = [ re_path(r'^admin/', admin.site.urls), # Django admin re_path(r'^api/', include('api.urls')), # Our API mirrored for session access - re_path(r'^lockss/', include('lockss.urls', namespace='lockss')), # Our app that communicates with the mirror network re_path(r'^', include('perma.urls')), # The Perma app ]