diff --git a/app/config/dartlang-pub-dev.yaml b/app/config/dartlang-pub-dev.yaml index c884366722..670ff4de85 100644 --- a/app/config/dartlang-pub-dev.yaml +++ b/app/config/dartlang-pub-dev.yaml @@ -7,7 +7,6 @@ projectId: dartlang-pub-dev searchServicePrefix: https://{{GAE_VERSION}}-dot-search-dot-dartlang-pub-dev.appspot.com fallbackSearchServicePrefix: https://search-dot-dartlang-pub-dev.appspot.com defaultServiceBaseUrl: https://{{GAE_VERSION}}-dot-{{GOOGLE_CLOUD_PROJECT}}.appspot.com -popularityDumpBucketName: dartlang-pub-dev--popularity searchSnapshotBucketName: dartlang-pub-dev--search-snapshot exportedApiBucketName: dartlang-pub-dev-exported-api maxTaskInstances: 50 diff --git a/app/config/dartlang-pub.yaml b/app/config/dartlang-pub.yaml index 4ab9de9f15..0a0a3dca06 100644 --- a/app/config/dartlang-pub.yaml +++ b/app/config/dartlang-pub.yaml @@ -10,7 +10,6 @@ projectId: dartlang-pub searchServicePrefix: https://{{GAE_VERSION}}-dot-search-dot-dartlang-pub.appspot.com fallbackSearchServicePrefix: https://search-dot-dartlang-pub.appspot.com defaultServiceBaseUrl: https://{{GAE_VERSION}}-dot-{{GOOGLE_CLOUD_PROJECT}}.appspot.com -popularityDumpBucketName: dartlang-pub--popularity searchSnapshotBucketName: dartlang-pub--search-snapshot exportedApiBucketName: dartlang-pub-exported-api maxTaskInstances: 700 diff --git a/app/lib/fake/backend/fake_popularity.dart b/app/lib/fake/backend/fake_popularity.dart index 5d116f4dfd..0970d0ca57 100644 --- a/app/lib/fake/backend/fake_popularity.dart +++ b/app/lib/fake/backend/fake_popularity.dart @@ -9,23 +9,6 @@ import 'package:clock/clock.dart'; import '../../package/models.dart'; import '../../service/download_counts/backend.dart'; import '../../shared/datastore.dart'; -import '../../shared/popularity_storage.dart'; - -/// Scans the datastore for packages and generates popularity values with a -/// deterministic random seed. -/// -/// TODO: generate similar values for download counts. -Future generateFakePopularityValues() async { - final values = {}; - final query = dbService.query(); - await for (final p in query.run()) { - final r = math.Random(p.name.hashCode.abs()); - final value = (math.min(p.likes * p.likes, 50) + r.nextInt(50)) / 100; - values[p.name!] = value; - } - // ignore: invalid_use_of_visible_for_testing_member - popularityStorage.updateValues(values, invalid: false); -} /// Scans the datastore for packages and generates download count values with a /// deterministic random seed. diff --git a/app/lib/fake/server/fake_analyzer_service.dart b/app/lib/fake/server/fake_analyzer_service.dart index 3fd760b50a..896c653dfe 100644 --- a/app/lib/fake/server/fake_analyzer_service.dart +++ b/app/lib/fake/server/fake_analyzer_service.dart @@ -39,7 +39,6 @@ class FakeAnalyzerService { cloudCompute: _cloudCompute, fn: () async { await generateFakeDownloadCounts(); - await generateFakePopularityValues(); final handler = wrapHandler(_logger, analyzerServiceHandler); final server = await IOServer.bind('localhost', port); diff --git a/app/lib/fake/server/fake_default_service.dart b/app/lib/fake/server/fake_default_service.dart index 461767c637..05e3e9d43d 100644 --- a/app/lib/fake/server/fake_default_service.dart +++ b/app/lib/fake/server/fake_default_service.dart @@ -49,7 +49,6 @@ class FakePubServer { } await generateFakeDownloadCounts(); - await generateFakePopularityValues(); await generateFakeTopicValues(); await nameTracker.startTracking(); diff --git a/app/lib/fake/server/fake_search_service.dart b/app/lib/fake/server/fake_search_service.dart index 2add5555fb..d70abb763e 100644 --- a/app/lib/fake/server/fake_search_service.dart +++ b/app/lib/fake/server/fake_search_service.dart @@ -55,7 +55,6 @@ class FakeSearchService { _logger.info('running on port $port'); await generateFakeDownloadCounts(); - await generateFakePopularityValues(); // ignore: invalid_use_of_visible_for_testing_member await indexUpdater.updateAllPackages(); diff --git a/app/lib/frontend/templates/package_misc.dart b/app/lib/frontend/templates/package_misc.dart index 10394462d6..d8b96e8319 100644 --- a/app/lib/frontend/templates/package_misc.dart +++ b/app/lib/frontend/templates/package_misc.dart @@ -196,7 +196,6 @@ d.Node labeledScoresNodeFromPackageView(PackageView view, {String? version}) { pkgScorePageUrl: urls.pkgScoreUrl(view.name, version: version), likeCount: view.likes, grantedPubPoints: view.grantedPubPoints, - popularity: view.popularity, thirtyDaysDownloads: view.thirtyDaysDownloadCounts, ); } diff --git a/app/lib/frontend/templates/views/pkg/labeled_scores.dart b/app/lib/frontend/templates/views/pkg/labeled_scores.dart index 1b487615ae..147278c7a8 100644 --- a/app/lib/frontend/templates/views/pkg/labeled_scores.dart +++ b/app/lib/frontend/templates/views/pkg/labeled_scores.dart @@ -10,7 +10,6 @@ d.Node labeledScoresNode({ required String pkgScorePageUrl, required int likeCount, required int? grantedPubPoints, - required int? popularity, required int? thirtyDaysDownloads, }) { return d.a( diff --git a/app/lib/package/models.dart b/app/lib/package/models.dart index 8146474381..c458e0dc83 100644 --- a/app/lib/package/models.dart +++ b/app/lib/package/models.dart @@ -19,7 +19,6 @@ import '../search/search_service.dart' show ApiPageRef; import '../shared/datastore.dart' as db; import '../shared/exceptions.dart'; import '../shared/model_properties.dart'; -import '../shared/popularity_storage.dart'; import '../shared/urls.dart' as urls; import '../shared/utils.dart'; @@ -946,7 +945,6 @@ class PackageView { final List? screenshots; final List? topics; - final int popularity; final int? thirtyDaysDownloadCounts; PackageView({ @@ -965,7 +963,6 @@ class PackageView { this.spdxIdentifiers, this.apiPages, this.topics, - required this.popularity, required this.thirtyDaysDownloadCounts, }) : isPending = isPending ?? false, tags = tags ?? []; @@ -979,7 +976,6 @@ class PackageView { PackageVersion? version, required ScoreCardData scoreCard, List? apiPages, - required int popularity, required int? thirtyDaysDownloadCounts, }) { final tags = { @@ -1004,7 +1000,6 @@ class PackageView { apiPages: apiPages, screenshots: scoreCard.panaReport?.screenshots, topics: version?.pubspec?.canonicalizedTopics, - popularity: popularity, thirtyDaysDownloadCounts: thirtyDaysDownloadCounts, ); } @@ -1026,7 +1021,6 @@ class PackageView { apiPages: apiPages ?? this.apiPages, screenshots: screenshots, topics: topics, - popularity: popularity, thirtyDaysDownloadCounts: thirtyDaysDownloadCounts, ); } @@ -1177,7 +1171,6 @@ class PackagePageData { releases: latestReleases, version: version, scoreCard: scoreCard, - popularity: popularityStorage.lookupAsScore(package.name!), thirtyDaysDownloadCounts: downloadCountsBackend.lookup30DaysTotalCounts(package.name!), ); diff --git a/app/lib/package/models.g.dart b/app/lib/package/models.g.dart index 0bb00d6098..a7bd2807a7 100644 --- a/app/lib/package/models.g.dart +++ b/app/lib/package/models.g.dart @@ -111,7 +111,6 @@ PackageView _$PackageViewFromJson(Map json) => PackageView( .toList(), topics: (json['topics'] as List?)?.map((e) => e as String).toList(), - popularity: (json['popularity'] as num).toInt(), thirtyDaysDownloadCounts: (json['thirtyDaysDownloadCounts'] as num?)?.toInt(), ); @@ -135,7 +134,6 @@ Map _$PackageViewToJson(PackageView instance) => if (instance.apiPages case final value?) 'apiPages': value, if (instance.screenshots case final value?) 'screenshots': value, if (instance.topics case final value?) 'topics': value, - 'popularity': instance.popularity, if (instance.thirtyDaysDownloadCounts case final value?) 'thirtyDaysDownloadCounts': value, }; diff --git a/app/lib/scorecard/backend.dart b/app/lib/scorecard/backend.dart index f0c022e682..540ce8fd08 100644 --- a/app/lib/scorecard/backend.dart +++ b/app/lib/scorecard/backend.dart @@ -12,7 +12,6 @@ import 'package:pool/pool.dart'; import 'package:pub_dev/service/download_counts/backend.dart'; import 'package:pub_dev/service/download_counts/computations.dart'; import 'package:pub_dev/shared/exceptions.dart'; -import 'package:pub_dev/shared/popularity_storage.dart'; import 'package:pub_dev/task/backend.dart'; import 'package:pub_dev/task/models.dart'; @@ -98,7 +97,6 @@ class ScoreCardBackend { releases: releases, version: pv, scoreCard: card, - popularity: popularityStorage.lookupAsScore(package), thirtyDaysDownloadCounts: downloadCountsBackend.lookup30DaysTotalCounts(package), ); diff --git a/app/lib/service/entrypoint/analyzer.dart b/app/lib/service/entrypoint/analyzer.dart index 3ef01f2d0d..21166faf10 100644 --- a/app/lib/service/entrypoint/analyzer.dart +++ b/app/lib/service/entrypoint/analyzer.dart @@ -15,7 +15,6 @@ import '../../analyzer/handlers.dart'; import '../../service/services.dart'; import '../../shared/env_config.dart'; import '../../shared/handler_helpers.dart'; -import '../../shared/popularity_storage.dart'; import '../../task/backend.dart'; import '../../tool/neat_task/pub_dev_tasks.dart'; @@ -63,7 +62,6 @@ class AnalyzerCommand extends Command { Future _workerMain(EntryMessage message) async { message.protocolSendPort.send(ReadyMessage()); - await popularityStorage.start(); await downloadCountsBackend.start(); await taskBackend.start(); registerScopeExitCallback(() => taskBackend.stop()); @@ -76,14 +74,12 @@ Future _workerMain(EntryMessage message) async { Future _indexBuilderMain(EntryMessage message) async { message.protocolSendPort.send(ReadyMessage()); - await popularityStorage.start(); await downloadCountsBackend.start(); await searchBackend.updateSnapshotInForeverLoop(); } Future _apiExporterMain(EntryMessage message) async { message.protocolSendPort.send(ReadyMessage()); - await popularityStorage.start(); await downloadCountsBackend.start(); await apiExporter!.start(); registerScopeExitCallback(() => apiExporter!.stop()); diff --git a/app/lib/service/entrypoint/frontend.dart b/app/lib/service/entrypoint/frontend.dart index 639fee9231..3f56fcb5e5 100644 --- a/app/lib/service/entrypoint/frontend.dart +++ b/app/lib/service/entrypoint/frontend.dart @@ -21,7 +21,6 @@ import '../../service/announcement/backend.dart'; import '../../service/youtube/backend.dart'; import '../../shared/env_config.dart'; import '../../shared/handler_helpers.dart'; -import '../../shared/popularity_storage.dart'; final Logger _logger = Logger('pub'); @@ -48,7 +47,6 @@ Future _main() async { if (envConfig.isRunningLocally) { await watchForResourceChanges(); } - await popularityStorage.start(); await nameTracker.startTracking(); await announcementBackend.start(); await topPackages.start(); diff --git a/app/lib/service/services.dart b/app/lib/service/services.dart index 075b5a983f..8068525db1 100644 --- a/app/lib/service/services.dart +++ b/app/lib/service/services.dart @@ -53,7 +53,6 @@ import '../shared/configuration.dart'; import '../shared/datastore.dart'; import '../shared/env_config.dart'; import '../shared/handler_helpers.dart'; -import '../shared/popularity_storage.dart'; import '../shared/redis_cache.dart' show setupCache; import '../shared/storage.dart'; import '../shared/versions.dart'; @@ -260,10 +259,6 @@ Future _withPubServices(FutureOr Function() fn) async { registerNameTracker(NameTracker(dbService)); registerPackageIndexHolder(PackageIndexHolder()); registerIndexUpdater(IndexUpdater(dbService)); - registerPopularityStorage( - PopularityStorage( - storageService.bucket(activeConfiguration.popularityDumpBucketName!)), - ); registerPublisherBackend(PublisherBackend(dbService)); registerScoreCardBackend(ScoreCardBackend(dbService)); registerSearchBackend(SearchBackend(dbService, @@ -297,7 +292,6 @@ Future _withPubServices(FutureOr Function() fn) async { registerScopeExitCallback(announcementBackend.close); registerScopeExitCallback(searchBackend.close); registerScopeExitCallback(() async => nameTracker.stopTracking()); - registerScopeExitCallback(popularityStorage.close); registerScopeExitCallback(scoreCardBackend.close); registerScopeExitCallback(searchClient.close); registerScopeExitCallback(topPackages.close); diff --git a/app/lib/shared/configuration.dart b/app/lib/shared/configuration.dart index 2fb986046a..66ee229772 100644 --- a/app/lib/shared/configuration.dart +++ b/app/lib/shared/configuration.dart @@ -103,9 +103,6 @@ final class Configuration { /// safe, as services may be deployed independently. final String defaultServiceBaseUrl; - /// The name of the Cloud Storage bucket to use for popularity data dumps. - final String? popularityDumpBucketName; - /// The name of the Cloud Storage bucket to use for search snapshots. final String? searchSnapshotBucketName; @@ -256,7 +253,6 @@ final class Configuration { required this.imageBucketName, required this.reportsBucketName, required this.downloadCountsBucketName, - required this.popularityDumpBucketName, required this.searchSnapshotBucketName, required this.exportedApiBucketName, required this.maxTaskInstances, @@ -321,7 +317,6 @@ final class Configuration { imageBucketName: 'fake-bucket-image', reportsBucketName: 'fake-bucket-reports', downloadCountsBucketName: 'fake-bucket-download-counts', - popularityDumpBucketName: 'fake-bucket-popularity', searchSnapshotBucketName: 'fake-bucket-search', exportedApiBucketName: 'fake-exported-apis', maxTaskInstances: 10, @@ -372,7 +367,6 @@ final class Configuration { imageBucketName: 'fake-bucket-image', reportsBucketName: 'fake-bucket-reports', downloadCountsBucketName: 'fake-bucket-download-counts', - popularityDumpBucketName: 'fake-bucket-popularity', searchSnapshotBucketName: 'fake-bucket-search', exportedApiBucketName: 'fake-exported-apis', taskResultBucketName: 'fake-bucket-task-result', @@ -419,7 +413,6 @@ final class Configuration { reportsBucketName!, downloadCountsBucketName!, incomingPackagesBucketName!, - popularityDumpBucketName!, publicPackagesBucketName!, searchSnapshotBucketName!, taskResultBucketName!, diff --git a/app/lib/shared/configuration.g.dart b/app/lib/shared/configuration.g.dart index c22cb14b36..591e84c608 100644 --- a/app/lib/shared/configuration.g.dart +++ b/app/lib/shared/configuration.g.dart @@ -24,7 +24,6 @@ Configuration _$ConfigurationFromJson(Map json) => $checkedCreate( 'searchServicePrefix', 'fallbackSearchServicePrefix', 'defaultServiceBaseUrl', - 'popularityDumpBucketName', 'searchSnapshotBucketName', 'maxTaskInstances', 'maxTaskRunHours', @@ -63,8 +62,6 @@ Configuration _$ConfigurationFromJson(Map json) => $checkedCreate( $checkedConvert('reportsBucketName', (v) => v as String?), downloadCountsBucketName: $checkedConvert('downloadCountsBucketName', (v) => v as String?), - popularityDumpBucketName: - $checkedConvert('popularityDumpBucketName', (v) => v as String?), searchSnapshotBucketName: $checkedConvert('searchSnapshotBucketName', (v) => v as String?), exportedApiBucketName: @@ -145,7 +142,6 @@ Map _$ConfigurationToJson(Configuration instance) => 'searchServicePrefix': instance.searchServicePrefix, 'fallbackSearchServicePrefix': instance.fallbackSearchServicePrefix, 'defaultServiceBaseUrl': instance.defaultServiceBaseUrl, - 'popularityDumpBucketName': instance.popularityDumpBucketName, 'searchSnapshotBucketName': instance.searchSnapshotBucketName, 'maxTaskInstances': instance.maxTaskInstances, 'maxTaskRunHours': instance.maxTaskRunHours, diff --git a/app/lib/shared/popularity_storage.dart b/app/lib/shared/popularity_storage.dart deleted file mode 100644 index ca04c2fa1b..0000000000 --- a/app/lib/shared/popularity_storage.dart +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import 'dart:async'; -import 'dart:convert'; -import 'dart:io'; - -import 'package:_popularity/popularity.dart'; -import 'package:clock/clock.dart'; -import 'package:gcloud/service_scope.dart' as ss; -import 'package:gcloud/storage.dart'; -import 'package:logging/logging.dart'; -import 'package:meta/meta.dart'; - -import '../shared/cached_value.dart'; -import '../shared/storage.dart'; - -final Logger _logger = Logger('pub.popularity'); -final GZipCodec _gzip = GZipCodec(); - -/// Sets the popularity storage -void registerPopularityStorage(PopularityStorage storage) => - ss.register(#_popularityStorage, storage); - -/// The active popularity storage -PopularityStorage get popularityStorage => - ss.lookup(#_popularityStorage) as PopularityStorage; - -class PopularityStorage { - late CachedValue<_PopularityData> _popularity; - late final _PopularityLoader _loader; - bool? _invalid; - - PopularityStorage(Bucket bucket) { - _loader = _PopularityLoader(bucket); - _popularity = CachedValue<_PopularityData>( - name: 'popularity', - // note: popularity data is stored in storage bucket, not cached in redis - interval: Duration(hours: 1), - maxAge: Duration(days: 14), - updateFn: () async => _loader.fetch(), - ); - } - - bool get isInvalid => - _invalid ?? - (!_popularity.isAvailable || (_popularity.value?.isInvalid ?? true)); - - DateTime? get lastFetched => _popularity.lastUpdated; - String? get dateRange => _popularity.value?.dateRange; - int get count => _popularity.value?.values.length ?? 0; - - double lookup(String package) => - _popularity.isAvailable ? _popularity.value!.values[package] ?? 0.0 : 0.0; - - int lookupAsScore(String package) => (lookup(package) * 100).round(); - - Future start() async { - await _popularity.update(); - } - - Future close() async { - await _popularity.close(); - } - - // Updates popularity scores to fixed values, useful for testing. - @visibleForTesting - void updateValues( - Map values, { - bool? invalid, - }) { - if (invalid != null) { - _invalid = invalid; - } - // ignore: invalid_use_of_visible_for_testing_member - _popularity.setValue( - _PopularityData(values: values, first: clock.now(), last: clock.now())); - } -} - -class _PopularityLoader { - final Bucket bucket; - ObjectInfo? _lastObjectInfo; - _PopularityData? _lastFetchedData; - - _PopularityLoader(this.bucket); - - Future<_PopularityData> fetch() async { - final objectName = PackagePopularity.popularityFileName; - _logger.info( - 'Checking popularity data info: ${bucketUri(bucket, objectName)}'); - final info = await bucket.info(objectName); - if (_lastFetchedData != null && - _lastObjectInfo != null && - _lastObjectInfo!.hasSameSignatureAs(info)) { - // Object didn't change since last fetch, returning the cached version. - return _lastFetchedData!; - } - _logger.info('Loading popularity data: ${bucketUri(bucket, objectName)}'); - final latest = (await bucket - .read(objectName) - .transform(_gzip.decoder) - .transform(utf8.decoder) - .transform(json.decoder) - .single) as Map; - final data = _processJson(latest); - _logger.info('Popularity updated for ${data.values.length} packages.'); - _lastObjectInfo = info; - _lastFetchedData = data; - return data; - } - - _PopularityData _processJson(Map raw) { - final popularity = PackagePopularity.fromJson(raw); - final List<_Entry> entries = <_Entry>[]; - popularity.items.forEach((package, totals) { - entries.add(_Entry(package, totals.score, totals.total)); - }); - entries.sort(); - final values = {}; - for (int i = 0; i < entries.length; i++) { - values[entries[i].package] = i / entries.length; - } - return _PopularityData( - values: values, - first: popularity.dateFirst, - last: popularity.dateLast, - ); - } -} - -class _PopularityData { - final Map values; - final DateTime? first; - final DateTime? last; - - _PopularityData({ - required this.values, - required this.first, - required this.last, - }); - - String get dateRange => - '${first?.toIso8601String()} - ${last?.toIso8601String()}'; - - /// Indicates that the data has no time range, or that the range is not - /// covering a full day, or that it is more than a month old. - late final isInvalid = first == null || - last == null || - last!.difference(first!).inDays < 1 || - clock.now().difference(last!).inDays > 30; -} - -class _Entry implements Comparable<_Entry> { - final String package; - final int score; - final int total; - - _Entry(this.package, this.score, this.total); - - @override - int compareTo(_Entry other) { - final int x = score.compareTo(other.score); - return x != 0 ? x : total.compareTo(other.total); - } -} diff --git a/app/test/shared/configuration_test.dart b/app/test/shared/configuration_test.dart index ce44007155..92b5f6d924 100644 --- a/app/test/shared/configuration_test.dart +++ b/app/test/shared/configuration_test.dart @@ -19,7 +19,6 @@ void main() { expect(config.imageBucketName, expectedValue); expect(config.reportsBucketName, expectedValue); expect(config.downloadCountsBucketName, expectedValue); - expect(config.popularityDumpBucketName, expectedValue); expect(config.admins![0].email, 'foo@foo.foo'); expect(config.admins![0].oauthUserId, '42'); expect(config.admins![0].permissions.contains(AdminPermission.listUsers), diff --git a/app/test/shared/test_data/foo_config.yaml b/app/test/shared/test_data/foo_config.yaml index be1cfa9881..80926c50a2 100644 --- a/app/test/shared/test_data/foo_config.yaml +++ b/app/test/shared/test_data/foo_config.yaml @@ -1,6 +1,5 @@ projectId: foo imageBucketName: foo -popularityDumpBucketName: foo searchSnapshotBucketName: foo canonicalPackagesBucketName: foo reportsBucketName: foo diff --git a/app/test/shared/test_services.dart b/app/test/shared/test_services.dart index dc008c832a..7d8ac4ad3a 100644 --- a/app/test/shared/test_services.dart +++ b/app/test/shared/test_services.dart @@ -111,7 +111,6 @@ class FakeAppengineEnv { await processTasksWithFakePanaAndDartdoc(); } await nameTracker.reloadFromDatastore(); - await generateFakePopularityValues(); await indexUpdater.updateAllPackages(); await topPackages.start(); await youtubeBackend.start(); @@ -213,7 +212,6 @@ void testWithFakeTime( ); await nameTracker.reloadFromDatastore(); await generateFakeDownloadCounts(); - await generateFakePopularityValues(); await indexUpdater.updateAllPackages(); await asyncQueue.ongoingProcessing; fakeEmailSender.sentMessages.clear();