From 365fe0b3d5a660b5a72e33facb5ed2a1fa293bc4 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Wed, 10 Jun 2020 15:54:34 -0700 Subject: [PATCH] python: Sort imports with isort. Fixes #2665. Regenerated by tabbott with `lint --fix` after a rebase and change in parameters. Note from tabbott: In a few cases, this converts technical debt in the form of unsorted imports into different technical debt in the form of our largest files having very long, ugly import sequences at the start. I expect this change will increase pressure for us to split those files, which isn't a bad thing. Signed-off-by: Anders Kaseorg --- analytics/lib/counts.py | 36 ++- analytics/lib/fixtures.py | 1 + analytics/lib/time_utils.py | 1 + .../commands/check_analytics_state.py | 3 +- .../commands/populate_analytics_db.py | 18 +- analytics/management/commands/realm_stats.py | 13 +- analytics/management/commands/stream_stats.py | 3 +- .../commands/update_analytics_counts.py | 2 +- .../migrations/0015_clear_duplicate_counts.py | 1 + analytics/models.py | 1 + analytics/tests/test_counts.py | 66 +++-- analytics/tests/test_fixtures.py | 1 + analytics/tests/test_views.py | 21 +- analytics/views.py | 67 +++-- confirmation/migrations/0001_initial.py | 2 +- .../migrations/0002_realmcreationkey.py | 2 +- .../migrations/0005_confirmation_realm.py | 3 +- confirmation/models.py | 19 +- corporate/lib/stripe.py | 29 +- corporate/models.py | 1 + corporate/tests/test_stripe.py | 81 ++++-- corporate/urls.py | 2 +- corporate/views.py | 47 +-- docs/conf.py | 3 +- frontend_tests/run-casper | 11 +- manage.py | 4 +- .../zulip_app_frontend/check_cron_file | 1 + .../check_send_receive_time | 11 +- .../check_postgres_replication_lag | 3 +- .../zulip_postgres_appdb/check_fts_update_log | 1 + .../files/postgresql/pg_backup_and_purge | 10 +- .../files/postgresql/process_fts_updates | 13 +- .../check_personal_zephyr_mirrors | 2 +- .../check_user_zephyr_mirror_liveness | 4 +- .../zulip_zephyr_mirror/check_zephyr_mirror | 2 +- .../files/zulip-ec2-configure-interfaces | 5 +- scripts/lib/check_rabbitmq_queue.py | 5 +- scripts/lib/clean-unused-caches | 3 +- scripts/lib/clean_emoji_cache.py | 10 +- scripts/lib/clean_node_cache.py | 10 +- scripts/lib/clean_venv_cache.py | 10 +- scripts/lib/create-production-venv | 9 +- scripts/lib/create-thumbor-venv | 9 +- scripts/lib/email-mirror-postfix | 17 +- scripts/lib/hash_reqs.py | 5 +- scripts/lib/node_cache.py | 6 +- scripts/lib/pythonrc.py | 5 +- scripts/lib/queue_workers.py | 2 +- scripts/lib/setup_path.py | 2 +- scripts/lib/setup_venv.py | 6 +- scripts/lib/sharding.py | 2 +- scripts/lib/unpack-zulip | 15 +- scripts/lib/upgrade-zulip | 20 +- scripts/lib/upgrade-zulip-from-git | 21 +- scripts/lib/upgrade-zulip-stage-2 | 12 +- scripts/lib/zulip_tools.py | 7 +- scripts/nagios/check-rabbitmq-consumers | 7 +- scripts/nagios/cron_file_helper.py | 2 +- scripts/purge-old-deployments | 5 +- scripts/restart-server | 8 +- scripts/setup/flush-memcached | 4 +- scripts/setup/generate_secrets.py | 7 +- scripts/setup/inline_email_css.py | 6 +- scripts/setup/restore-backup | 4 +- scripts/zulip-puppet-apply | 10 +- static/assets/favicon/generate | 2 +- tools/check-capitalization | 4 +- tools/check-frontend-i18n | 8 +- tools/check-issue-labels | 10 +- tools/check-provision | 8 +- tools/check-templates | 12 +- tools/check-thirdparty | 2 +- tools/create-test-api-docs | 7 +- tools/diagnose | 7 +- .../commands/crawl_with_status.py | 5 +- .../spiders/check_documentation.py | 1 - .../spiders/check_help_documentation.py | 4 +- .../spiders/common/spiders.py | 5 +- tools/droplets/add_mentor.py | 5 +- tools/droplets/create.py | 15 +- tools/fetch-contributor-data | 17 +- tools/generate-integration-docs-screenshot | 25 +- tools/i18n/process-mobile-i18n | 1 + tools/i18n/tagmessages | 3 +- tools/lib/capitalization.py | 2 +- tools/lib/gitlint-rules.py | 6 +- tools/lib/html_branches.py | 9 +- tools/lib/pretty_print.py | 9 +- tools/lib/provision.py | 29 +- tools/lib/provision_inner.py | 23 +- tools/lib/sanity_check.py | 1 + tools/lib/template_parser.py | 1 + tools/lib/test_script.py | 8 +- tools/lib/test_server.py | 6 +- tools/lint | 17 +- tools/linter_lib/pep8.py | 3 +- tools/linter_lib/pyflakes.py | 1 - tools/pretty-print-html | 4 +- tools/renumber-migrations | 7 +- tools/review | 2 +- tools/run-dev.py | 16 +- tools/run-mypy | 6 +- tools/setup/build_pygments_data | 4 +- tools/setup/emoji/build_emoji | 16 +- tools/setup/emoji/emoji_setup_utils.py | 2 - tools/setup/emoji/export_emoji_names_to_csv | 4 +- tools/setup/emoji/generate_emoji_names_table | 6 +- tools/setup/emoji/import_emoji_names_from_csv | 1 - tools/setup/emoji/test-emoji-name-scripts | 2 +- .../generate_integration_bots_avatars.py | 5 +- .../setup/generate_zulip_bots_static_files.py | 5 +- tools/show-profile-results | 2 +- tools/test-api | 9 +- tools/test-backend | 23 +- tools/test-help-documentation | 3 +- tools/test-js-with-node | 6 +- tools/test-js-with-puppeteer | 14 +- tools/test-locked-requirements | 4 +- tools/test-queue-worker-reload | 7 +- tools/test-run-dev | 2 +- tools/test-tools | 3 +- tools/tests/test_capitalization_checker.py | 7 +- tools/tests/test_check_rabbitmq_queue.py | 14 +- tools/tests/test_html_branches.py | 5 +- tools/tests/test_template_parser.py | 3 +- tools/tests/test_zulint_custom_rules.py | 8 +- tools/update-prod-static | 9 +- tools/webpack | 3 +- tools/zulip-export/zulip-export | 6 +- zerver/apps.py | 1 + zerver/context_processors.py | 30 +- zerver/data_import/gitter.py | 29 +- zerver/data_import/hipchat.py | 30 +- zerver/data_import/hipchat_attachment.py | 9 +- zerver/data_import/hipchat_user.py | 9 +- zerver/data_import/import_util.py | 23 +- zerver/data_import/mattermost.py | 41 +-- zerver/data_import/mattermost_user.py | 1 + zerver/data_import/slack.py | 62 ++-- .../data_import/slack_message_conversion.py | 2 +- zerver/decorator.py | 74 ++--- zerver/filters.py | 1 + zerver/forms.py | 52 ++-- zerver/lib/actions.py | 268 +++++++++++------- zerver/lib/addressee.py | 6 +- zerver/lib/alert_words.py | 15 +- zerver/lib/attachments.py | 4 +- zerver/lib/avatar.py | 15 +- zerver/lib/avatar_hash.py | 4 +- zerver/lib/bot_config.py | 17 +- zerver/lib/bot_lib.py | 28 +- zerver/lib/bot_storage.py | 6 +- zerver/lib/bugdown/__init__.py | 83 +++--- .../bugdown/api_arguments_table_generator.py | 9 +- .../api_return_values_table_generator.py | 5 +- zerver/lib/bugdown/fenced_code.py | 5 +- .../help_emoticon_translations_table.py | 3 +- zerver/lib/bugdown/help_relative_links.py | 3 +- zerver/lib/bugdown/help_settings_links.py | 3 +- zerver/lib/bugdown/include.py | 4 +- zerver/lib/bugdown/nested_code_blocks.py | 10 +- zerver/lib/bugdown/tabbed_sections.py | 4 +- zerver/lib/bugdown/testing_mocks.py | 2 +- zerver/lib/bulk_create.py | 11 +- zerver/lib/cache.py | 43 +-- zerver/lib/cache_helpers.py | 40 ++- zerver/lib/camo.py | 4 +- zerver/lib/ccache.py | 7 +- zerver/lib/context_managers.py | 5 +- zerver/lib/create_user.py | 20 +- zerver/lib/db.py | 7 +- zerver/lib/debug.py | 2 +- zerver/lib/dev_ldap_directory.py | 2 + zerver/lib/digest.py | 26 +- zerver/lib/display_recipient.py | 11 +- zerver/lib/domains.py | 6 +- zerver/lib/email_mirror.py | 49 ++-- zerver/lib/email_mirror_helpers.py | 2 +- zerver/lib/email_notifications.py | 45 +-- zerver/lib/email_validation.py | 11 +- zerver/lib/emoji.py | 6 +- zerver/lib/error_notify.py | 9 +- zerver/lib/events.py | 56 ++-- zerver/lib/exceptions.py | 3 +- zerver/lib/export.py | 66 +++-- zerver/lib/external_accounts.py | 8 +- zerver/lib/fix_unreads.py | 4 +- zerver/lib/generate_test_data.py | 8 +- zerver/lib/hotspots.py | 6 +- zerver/lib/html_diff.py | 5 +- zerver/lib/html_to_text.py | 1 + zerver/lib/i18n.py | 13 +- zerver/lib/import_realm.py | 72 +++-- zerver/lib/initial_password.py | 7 +- zerver/lib/integrations.py | 6 +- zerver/lib/logging_util.py | 10 +- zerver/lib/management.py | 7 +- zerver/lib/mdiff.py | 3 +- zerver/lib/mention.py | 3 +- zerver/lib/message.py | 56 ++-- zerver/lib/migrate.py | 6 +- zerver/lib/mobile_auth_otp.py | 3 +- zerver/lib/name_restrictions.py | 1 + zerver/lib/narrow.py | 8 +- zerver/lib/onboarding.py | 14 +- zerver/lib/outgoing_webhook.py | 22 +- zerver/lib/parallel.py | 5 +- zerver/lib/presence.py | 15 +- zerver/lib/profile.py | 3 +- zerver/lib/push_notifications.py | 31 +- zerver/lib/pysa.py | 1 - zerver/lib/queue.py | 6 +- zerver/lib/rate_limiter.py | 10 +- zerver/lib/realm_description.py | 10 +- zerver/lib/realm_icon.py | 1 + zerver/lib/realm_logo.py | 1 + zerver/lib/redis_utils.py | 7 +- zerver/lib/remote_server.py | 5 +- zerver/lib/request.py | 12 +- zerver/lib/response.py | 6 +- zerver/lib/rest.py | 16 +- zerver/lib/retention.py | 28 +- zerver/lib/send_email.py | 32 +-- zerver/lib/server_initialization.py | 6 +- zerver/lib/sessions.py | 8 +- zerver/lib/soft_deactivation.py | 22 +- zerver/lib/sqlalchemy_utils.py | 5 +- zerver/lib/statistics.py | 5 +- zerver/lib/stream_recipient.py | 4 +- zerver/lib/stream_subscription.py | 9 +- zerver/lib/stream_topic.py | 9 +- zerver/lib/streams.py | 24 +- zerver/lib/subdomains.py | 6 +- zerver/lib/test_classes.py | 86 +++--- zerver/lib/test_fixtures.py | 20 +- zerver/lib/test_helpers.py | 51 ++-- zerver/lib/test_runner.py | 35 +-- zerver/lib/tex.py | 5 +- zerver/lib/thumbnail.py | 6 +- zerver/lib/timeout.py | 9 +- zerver/lib/timestamp.py | 3 +- zerver/lib/timezone.py | 5 +- zerver/lib/topic.py | 21 +- zerver/lib/topic_mutes.py | 23 +- zerver/lib/transfer.py | 8 +- zerver/lib/type_debug.py | 4 +- zerver/lib/types.py | 5 +- zerver/lib/unminify.py | 6 +- zerver/lib/upload.py | 64 ++--- zerver/lib/url_encoding.py | 1 + zerver/lib/url_preview/oembed.py | 6 +- zerver/lib/url_preview/parsers/__init__.py | 2 +- zerver/lib/url_preview/parsers/base.py | 1 + zerver/lib/url_preview/parsers/generic.py | 1 + zerver/lib/url_preview/parsers/open_graph.py | 1 + zerver/lib/url_preview/preview.py | 10 +- zerver/lib/user_groups.py | 7 +- zerver/lib/user_status.py | 7 +- zerver/lib/users.py | 27 +- zerver/lib/utils.py | 4 +- zerver/lib/validator.py | 12 +- zerver/lib/webhooks/common.py | 18 +- zerver/lib/webhooks/git.py | 3 +- zerver/lib/widget.py | 5 +- zerver/lib/zcommand.py | 4 +- zerver/lib/zephyr.py | 2 + zerver/logging_handlers.py | 6 +- zerver/management/commands/check_redis.py | 3 +- .../commands/convert_gitter_data.py | 3 +- .../commands/convert_hipchat_data.py | 3 +- .../commands/convert_mattermost_data.py | 3 +- .../management/commands/convert_slack_data.py | 3 +- .../commands/generate_invite_links.py | 2 +- zerver/management/commands/import.py | 3 +- zerver/management/commands/knight.py | 2 +- .../management/commands/logout_all_users.py | 7 +- zerver/management/commands/makemessages.py | 2 +- zerver/management/commands/merge_streams.py | 7 +- .../management/commands/restore_messages.py | 7 +- zerver/management/commands/runtornado.py | 14 +- zerver/management/commands/show_admins.py | 2 + .../commands/soft_deactivate_users.py | 8 +- .../commands/transfer_uploads_to_s3.py | 3 +- zerver/middleware.py | 7 +- zerver/migrations/0030_realm_org_type.py | 1 - .../0032_verify_all_medium_avatar_images.py | 2 +- ...0038_realm_change_to_community_defaults.py | 1 - .../0060_move_avatars_to_be_uid_based.py | 1 + .../0064_sync_uploads_filesize_with_db.py | 1 + ...0077_add_file_name_field_to_realm_emoji.py | 1 + .../0086_realm_alter_default_org_type.py | 1 - .../0143_realm_bot_creation_policy.py | 2 - zerver/migrations/0185_realm_plan_type.py | 2 - .../0214_realm_invite_to_stream_policy.py | 2 - ...228_userprofile_demote_inactive_streams.py | 2 - ..._userprofile_desktop_icon_count_display.py | 2 - .../migrations/0248_userprofile_role_start.py | 2 +- .../0252_realm_user_group_edit_policy.py | 2 - .../0261_realm_private_message_policy.py | 2 - .../migrations/0262_mutedtopic_date_muted.py | 2 +- .../0263_stream_stream_post_policy.py | 2 - zerver/migrations/0266_userpresence_realm.py | 3 +- ..._add_userpresence_realm_timestamp_index.py | 3 +- zerver/migrations/0270_huddle_recipient.py | 3 +- .../0273_migrate_old_bot_messages.py | 3 +- zerver/migrations/0276_alertword.py | 2 +- zerver/migrations/0277_migrate_alert_word.py | 6 +- ...84_convert_realm_admins_to_realm_owners.py | 3 +- zerver/models.py | 124 +++++--- zerver/openapi/curl_param_value_generators.py | 9 +- zerver/openapi/javascript_examples.py | 4 +- zerver/openapi/markdown_extension.py | 11 +- zerver/openapi/python_examples.py | 10 +- zerver/openapi/test_curl_examples.py | 13 +- zerver/signals.py | 5 +- zerver/templatetags/app_filters.py | 14 +- zerver/tests/test_alert_words.py | 28 +- zerver/tests/test_archive.py | 11 +- zerver/tests/test_attachments.py | 3 +- zerver/tests/test_audit_log.py | 38 ++- zerver/tests/test_auth_backends.py | 155 ++++++---- zerver/tests/test_bots.py | 36 ++- zerver/tests/test_bugdown.py | 49 ++-- zerver/tests/test_cache.py | 31 +- zerver/tests/test_camo.py | 1 + zerver/tests/test_compatibility.py | 2 +- zerver/tests/test_create_video_call.py | 1 + zerver/tests/test_custom_profile_data.py | 28 +- zerver/tests/test_decorators.py | 94 +++--- zerver/tests/test_digest.py | 24 +- zerver/tests/test_docs.py | 11 +- zerver/tests/test_email_change.py | 19 +- zerver/tests/test_email_log.py | 2 + zerver/tests/test_email_mirror.py | 65 ++--- zerver/tests/test_email_notifications.py | 28 +- zerver/tests/test_embedded_bot_system.py | 10 +- zerver/tests/test_event_queue.py | 22 +- zerver/tests/test_events.py | 113 +++++--- zerver/tests/test_external.py | 22 +- zerver/tests/test_gitter_importer.py | 27 +- zerver/tests/test_hipchat_importer.py | 18 +- zerver/tests/test_home.py | 34 ++- zerver/tests/test_hotspots.py | 7 +- zerver/tests/test_i18n.py | 19 +- zerver/tests/test_import_export.py | 99 +++---- zerver/tests/test_integrations.py | 14 +- zerver/tests/test_integrations_dev_panel.py | 7 +- zerver/tests/test_internet.py | 5 +- zerver/tests/test_legacy_subject.py | 5 +- zerver/tests/test_link_embed.py | 18 +- zerver/tests/test_logging_handlers.py | 10 +- zerver/tests/test_management_commands.py | 28 +- zerver/tests/test_mattermost_importer.py | 41 +-- .../tests/test_message_edit_notifications.py | 20 +- zerver/tests/test_messages.py | 117 ++++---- zerver/tests/test_middleware.py | 4 +- zerver/tests/test_migrations.py | 3 +- zerver/tests/test_muting.py | 16 +- zerver/tests/test_narrow.py | 74 +++-- zerver/tests/test_new_users.py | 12 +- zerver/tests/test_onboarding.py | 7 +- zerver/tests/test_openapi.py | 45 ++- .../tests/test_outgoing_webhook_interfaces.py | 19 +- zerver/tests/test_outgoing_webhook_system.py | 22 +- zerver/tests/test_populate_db.py | 5 +- zerver/tests/test_presence.py | 23 +- zerver/tests/test_push_notifications.py | 76 ++--- zerver/tests/test_queue.py | 8 +- zerver/tests/test_queue_worker.py | 17 +- zerver/tests/test_rate_limiter.py | 14 +- zerver/tests/test_reactions.py | 12 +- zerver/tests/test_realm.py | 37 ++- zerver/tests/test_realm_domains.py | 18 +- zerver/tests/test_realm_emoji.py | 4 +- zerver/tests/test_realm_export.py | 22 +- zerver/tests/test_realm_filters.py | 1 + zerver/tests/test_redis_utils.py | 11 +- zerver/tests/test_report.py | 9 +- zerver/tests/test_retention.py | 38 ++- zerver/tests/test_service_bot_system.py | 21 +- zerver/tests/test_sessions.py | 19 +- zerver/tests/test_settings.py | 14 +- zerver/tests/test_signup.py | 133 +++++---- zerver/tests/test_slack_importer.py | 87 +++--- zerver/tests/test_slack_message_conversion.py | 14 +- zerver/tests/test_soft_deactivation.py | 13 +- zerver/tests/test_subdomains.py | 3 +- zerver/tests/test_submessage.py | 13 +- zerver/tests/test_subs.py | 132 ++++----- zerver/tests/test_templates.py | 4 +- zerver/tests/test_thumbnail.py | 13 +- zerver/tests/test_timestamp.py | 18 +- zerver/tests/test_tornado.py | 14 +- zerver/tests/test_transfer.py | 24 +- zerver/tests/test_tutorial.py | 4 +- zerver/tests/test_type_debug.py | 5 +- zerver/tests/test_typing.py | 20 +- zerver/tests/test_unread.py | 29 +- zerver/tests/test_upload.py | 105 ++++--- zerver/tests/test_urls.py | 7 +- zerver/tests/test_user_groups.py | 17 +- zerver/tests/test_user_status.py | 24 +- zerver/tests/test_users.py | 84 +++--- zerver/tests/test_webhooks_common.py | 20 +- zerver/tests/test_widgets.py | 10 +- zerver/tests/test_zcommand.py | 5 +- zerver/tests/test_zephyr.py | 8 +- zerver/tornado/application.py | 3 +- zerver/tornado/autoreload.py | 9 +- zerver/tornado/descriptors.py | 2 +- zerver/tornado/event_queue.py | 64 +++-- zerver/tornado/exceptions.py | 1 + zerver/tornado/handlers.py | 2 +- zerver/tornado/sharding.py | 6 +- zerver/tornado/views.py | 18 +- zerver/views/alert_words.py | 12 +- zerver/views/archive.py | 12 +- zerver/views/attachments.py | 5 +- zerver/views/auth.py | 83 +++--- zerver/views/camo.py | 8 +- zerver/views/compatibility.py | 5 +- zerver/views/custom_profile_fields.py | 44 +-- zerver/views/development/email_log.py | 26 +- zerver/views/development/integrations.py | 12 +- zerver/views/development/registration.py | 9 +- zerver/views/digest.py | 8 +- zerver/views/documentation.py | 20 +- zerver/views/email_mirror.py | 6 +- zerver/views/events_register.py | 6 +- zerver/views/home.py | 33 +-- zerver/views/hotspots.py | 3 +- zerver/views/invite.py | 26 +- zerver/views/messages.py | 135 ++++++--- zerver/views/muting.py | 12 +- zerver/views/pointer.py | 3 +- zerver/views/portico.py | 5 +- zerver/views/presence.py | 21 +- zerver/views/push_notifications.py | 10 +- zerver/views/reactions.py | 6 +- zerver/views/realm.py | 35 ++- zerver/views/realm_domains.py | 5 +- zerver/views/realm_emoji.py | 10 +- zerver/views/realm_export.py | 13 +- zerver/views/realm_filters.py | 6 +- zerver/views/realm_icon.py | 2 +- zerver/views/realm_logo.py | 6 +- zerver/views/registration.py | 113 +++++--- zerver/views/report.py | 11 +- zerver/views/storage.py | 14 +- zerver/views/streams.py | 96 +++++-- zerver/views/submessage.py | 12 +- zerver/views/thumbnail.py | 11 +- zerver/views/tutorial.py | 3 +- zerver/views/typing.py | 6 +- zerver/views/unsubscribe.py | 11 +- zerver/views/upload.py | 25 +- zerver/views/user_groups.py | 34 ++- zerver/views/user_settings.py | 54 ++-- zerver/views/users.py | 92 ++++-- zerver/views/video_calls.py | 6 +- zerver/views/zephyr.py | 23 +- zerver/webhooks/alertmanager/view.py | 1 + zerver/webhooks/ansibletower/view.py | 3 +- zerver/webhooks/appveyor/view.py | 3 +- zerver/webhooks/basecamp/view.py | 3 +- zerver/webhooks/beanstalk/tests.py | 1 - zerver/webhooks/beanstalk/view.py | 3 +- zerver/webhooks/bitbucket/tests.py | 1 - zerver/webhooks/bitbucket/view.py | 3 +- zerver/webhooks/bitbucket2/view.py | 23 +- zerver/webhooks/bitbucket3/view.py | 25 +- zerver/webhooks/clubhouse/tests.py | 1 - zerver/webhooks/clubhouse/view.py | 3 +- zerver/webhooks/flock/view.py | 3 +- zerver/webhooks/github/view.py | 27 +- zerver/webhooks/gitlab/view.py | 22 +- zerver/webhooks/gogs/view.py | 24 +- zerver/webhooks/gosquared/view.py | 3 +- zerver/webhooks/grafana/view.py | 2 +- zerver/webhooks/groove/view.py | 9 +- zerver/webhooks/harbor/view.py | 3 +- zerver/webhooks/hellosign/tests.py | 4 +- zerver/webhooks/hellosign/view.py | 2 +- zerver/webhooks/insping/view.py | 3 +- zerver/webhooks/intercom/view.py | 3 +- zerver/webhooks/jira/view.py | 3 +- zerver/webhooks/netlify/view.py | 9 +- zerver/webhooks/newrelic/view.py | 3 +- zerver/webhooks/pagerduty/view.py | 3 +- zerver/webhooks/pingdom/view.py | 3 +- zerver/webhooks/pivotal/view.py | 3 +- zerver/webhooks/raygun/view.py | 3 +- zerver/webhooks/reviewboard/view.py | 9 +- zerver/webhooks/semaphore/tests.py | 4 +- zerver/webhooks/semaphore/view.py | 3 +- zerver/webhooks/sentry/view.py | 3 +- zerver/webhooks/slack_incoming/view.py | 9 +- zerver/webhooks/statuspage/view.py | 3 +- zerver/webhooks/stripe/view.py | 3 +- zerver/webhooks/teamcity/tests.py | 3 +- zerver/webhooks/teamcity/view.py | 6 +- zerver/webhooks/transifex/view.py | 3 +- zerver/webhooks/trello/view/__init__.py | 9 +- zerver/webhooks/updown/view.py | 3 +- zerver/webhooks/zabbix/view.py | 3 +- zerver/worker/queue_processors.py | 122 +++++--- zilencer/forms.py | 1 + .../commands/add_mock_conversation.py | 12 +- zilencer/management/commands/add_new_realm.py | 3 +- zilencer/management/commands/populate_db.py | 55 ++-- zilencer/models.py | 3 +- zilencer/views.py | 39 ++- zproject/backends.py | 84 +++--- zproject/computed_settings.py | 19 +- zproject/config.py | 2 +- zproject/configured_settings.py | 1 + zproject/default_settings.py | 5 +- zproject/dev_settings.py | 1 + zproject/dev_urls.py | 8 +- zproject/email_backends.py | 9 +- zproject/jinja2/__init__.py | 2 +- zproject/legacy_urls.py | 5 +- zproject/test_extra_settings.py | 1 + zproject/test_settings.py | 1 + zproject/urls.py | 56 ++-- zproject/wsgi.py | 3 + zthumbor/loaders/helpers.py | 2 +- zthumbor/loaders/zloader.py | 16 +- 528 files changed, 4670 insertions(+), 3675 deletions(-) diff --git a/analytics/lib/counts.py b/analytics/lib/counts.py index f83346a8e182f..a3f9ee84068c3 100644 --- a/analytics/lib/counts.py +++ b/analytics/lib/counts.py @@ -1,23 +1,35 @@ +import logging import time from collections import OrderedDict, defaultdict from datetime import datetime, timedelta -import logging -from typing import Callable, Dict, List, \ - Optional, Tuple, Type, Union +from typing import Callable, Dict, List, Optional, Tuple, Type, Union from django.conf import settings from django.db import connection from django.db.models import F -from psycopg2.sql import Composable, Identifier, Literal, SQL - -from analytics.models import BaseCount, \ - FillState, InstallationCount, RealmCount, StreamCount, \ - UserCount, installation_epoch, last_successful_fill +from psycopg2.sql import SQL, Composable, Identifier, Literal + +from analytics.models import ( + BaseCount, + FillState, + InstallationCount, + RealmCount, + StreamCount, + UserCount, + installation_epoch, + last_successful_fill, +) from zerver.lib.logging_util import log_to_file -from zerver.lib.timestamp import ceiling_to_day, \ - ceiling_to_hour, floor_to_hour, verify_UTC -from zerver.models import Message, Realm, RealmAuditLog, \ - Stream, UserActivityInterval, UserProfile, models +from zerver.lib.timestamp import ceiling_to_day, ceiling_to_hour, floor_to_hour, verify_UTC +from zerver.models import ( + Message, + Realm, + RealmAuditLog, + Stream, + UserActivityInterval, + UserProfile, + models, +) ## Logging setup ## diff --git a/analytics/lib/fixtures.py b/analytics/lib/fixtures.py index a439b56d95c5d..c707220a1932d 100644 --- a/analytics/lib/fixtures.py +++ b/analytics/lib/fixtures.py @@ -4,6 +4,7 @@ from analytics.lib.counts import CountStat + def generate_time_series_data(days: int=100, business_hours_base: float=10, non_business_hours_base: float=10, growth: float=1, autocorrelation: float=0, spikiness: float=1, diff --git a/analytics/lib/time_utils.py b/analytics/lib/time_utils.py index 48d8f54223797..b990ae252d545 100644 --- a/analytics/lib/time_utils.py +++ b/analytics/lib/time_utils.py @@ -4,6 +4,7 @@ from analytics.lib.counts import CountStat from zerver.lib.timestamp import floor_to_day, floor_to_hour, verify_UTC + # If min_length is None, returns end_times from ceiling(start) to floor(end), inclusive. # If min_length is greater than 0, pads the list to the left. # So informally, time_range(Sep 20, Sep 22, day, None) returns [Sep 20, Sep 21, Sep 22], diff --git a/analytics/management/commands/check_analytics_state.py b/analytics/management/commands/check_analytics_state.py index c1a14f0341f35..698317b6f50f3 100644 --- a/analytics/management/commands/check_analytics_state.py +++ b/analytics/management/commands/check_analytics_state.py @@ -8,8 +8,7 @@ from analytics.lib.counts import COUNT_STATS, CountStat from analytics.models import installation_epoch, last_successful_fill -from zerver.lib.timestamp import TimezoneNotUTCException, floor_to_day, \ - floor_to_hour, verify_UTC +from zerver.lib.timestamp import TimezoneNotUTCException, floor_to_day, floor_to_hour, verify_UTC from zerver.models import Realm states = { diff --git a/analytics/management/commands/populate_analytics_db.py b/analytics/management/commands/populate_analytics_db.py index 462f41447b772..70857dc6306ae 100644 --- a/analytics/management/commands/populate_analytics_db.py +++ b/analytics/management/commands/populate_analytics_db.py @@ -1,21 +1,25 @@ from datetime import timedelta from typing import Any, Dict, List, Mapping, Optional, Type - from unittest import mock + from django.core.management.base import BaseCommand from django.utils.timezone import now as timezone_now -from analytics.lib.counts import COUNT_STATS, CountStat, \ - do_drop_all_analytics_tables +from analytics.lib.counts import COUNT_STATS, CountStat, do_drop_all_analytics_tables from analytics.lib.fixtures import generate_time_series_data from analytics.lib.time_utils import time_range -from analytics.models import BaseCount, FillState, InstallationCount, \ - RealmCount, StreamCount, UserCount +from analytics.models import ( + BaseCount, + FillState, + InstallationCount, + RealmCount, + StreamCount, + UserCount, +) from zerver.lib.actions import STREAM_ASSIGNMENT_COLORS, do_change_user_role from zerver.lib.create_user import create_user from zerver.lib.timestamp import floor_to_day -from zerver.models import Client, Realm, Recipient, Stream, Subscription, \ - UserProfile +from zerver.models import Client, Realm, Recipient, Stream, Subscription, UserProfile class Command(BaseCommand): diff --git a/analytics/management/commands/realm_stats.py b/analytics/management/commands/realm_stats.py index 8730a3a217f29..336a092f6ce70 100644 --- a/analytics/management/commands/realm_stats.py +++ b/analytics/management/commands/realm_stats.py @@ -6,8 +6,17 @@ from django.db.models import Count from django.utils.timezone import now as timezone_now -from zerver.models import Message, Realm, Recipient, Stream, Subscription, \ - UserActivity, UserMessage, UserProfile, get_realm +from zerver.models import ( + Message, + Realm, + Recipient, + Stream, + Subscription, + UserActivity, + UserMessage, + UserProfile, + get_realm, +) MOBILE_CLIENT_LIST = ["Android", "ios"] HUMAN_CLIENT_LIST = MOBILE_CLIENT_LIST + ["website"] diff --git a/analytics/management/commands/stream_stats.py b/analytics/management/commands/stream_stats.py index b6bde4bdc2636..4bb861c00cb9d 100644 --- a/analytics/management/commands/stream_stats.py +++ b/analytics/management/commands/stream_stats.py @@ -4,8 +4,7 @@ from django.core.management.base import BaseCommand, CommandError from django.db.models import Q -from zerver.models import Message, Realm, Recipient, Stream, Subscription, \ - get_realm +from zerver.models import Message, Realm, Recipient, Stream, Subscription, get_realm class Command(BaseCommand): diff --git a/analytics/management/commands/update_analytics_counts.py b/analytics/management/commands/update_analytics_counts.py index 9e89bb2246725..2f66409563314 100644 --- a/analytics/management/commands/update_analytics_counts.py +++ b/analytics/management/commands/update_analytics_counts.py @@ -1,8 +1,8 @@ import os import time from argparse import ArgumentParser -from typing import Any, Dict from datetime import timezone +from typing import Any, Dict from django.conf import settings from django.core.management.base import BaseCommand diff --git a/analytics/migrations/0015_clear_duplicate_counts.py b/analytics/migrations/0015_clear_duplicate_counts.py index b97f8265fa975..9533f21bdef60 100644 --- a/analytics/migrations/0015_clear_duplicate_counts.py +++ b/analytics/migrations/0015_clear_duplicate_counts.py @@ -3,6 +3,7 @@ from django.db.migrations.state import StateApps from django.db.models import Count, Sum + def clear_duplicate_counts(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: """This is a preparatory migration for our Analytics tables. diff --git a/analytics/models.py b/analytics/models.py index b81ea5cff1b5c..20bcc8b6a6ed7 100644 --- a/analytics/models.py +++ b/analytics/models.py @@ -7,6 +7,7 @@ from zerver.lib.timestamp import floor_to_day from zerver.models import Realm, Stream, UserProfile + class FillState(models.Model): property: str = models.CharField(max_length=40, unique=True) end_time: datetime.datetime = models.DateTimeField() diff --git a/analytics/tests/test_counts.py b/analytics/tests/test_counts.py index 7992da7af6e04..52f58f6f5a007 100644 --- a/analytics/tests/test_counts.py +++ b/analytics/tests/test_counts.py @@ -1,7 +1,7 @@ from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional, Tuple, Type - from unittest import mock + import ujson from django.apps import apps from django.db import models @@ -10,24 +10,58 @@ from django.utils.timezone import now as timezone_now from psycopg2.sql import SQL, Literal -from analytics.lib.counts import COUNT_STATS, CountStat, get_count_stats, \ - DependentCountStat, LoggingCountStat, do_aggregate_to_summary_table, \ - do_drop_all_analytics_tables, do_drop_single_stat, \ - do_fill_count_stat_at_hour, do_increment_logging_stat, \ - process_count_stat, sql_data_collector -from analytics.models import BaseCount, \ - FillState, InstallationCount, RealmCount, StreamCount, \ - UserCount, installation_epoch -from zerver.lib.actions import do_activate_user, do_create_user, \ - do_deactivate_user, do_reactivate_user, update_user_activity_interval, \ - do_invite_users, do_revoke_user_invite, do_resend_user_invite_email, \ - InvitationError +from analytics.lib.counts import ( + COUNT_STATS, + CountStat, + DependentCountStat, + LoggingCountStat, + do_aggregate_to_summary_table, + do_drop_all_analytics_tables, + do_drop_single_stat, + do_fill_count_stat_at_hour, + do_increment_logging_stat, + get_count_stats, + process_count_stat, + sql_data_collector, +) +from analytics.models import ( + BaseCount, + FillState, + InstallationCount, + RealmCount, + StreamCount, + UserCount, + installation_epoch, +) +from zerver.lib.actions import ( + InvitationError, + do_activate_user, + do_create_user, + do_deactivate_user, + do_invite_users, + do_reactivate_user, + do_resend_user_invite_email, + do_revoke_user_invite, + update_user_activity_interval, +) from zerver.lib.create_user import create_user from zerver.lib.timestamp import TimezoneNotUTCException, floor_to_day from zerver.lib.topic import DB_TOPIC_NAME -from zerver.models import Client, Huddle, Message, Realm, \ - RealmAuditLog, Recipient, Stream, UserActivityInterval, \ - UserProfile, get_client, get_user, PreregistrationUser +from zerver.models import ( + Client, + Huddle, + Message, + PreregistrationUser, + Realm, + RealmAuditLog, + Recipient, + Stream, + UserActivityInterval, + UserProfile, + get_client, + get_user, +) + class AnalyticsTestCase(TestCase): MINUTE = timedelta(seconds = 60) diff --git a/analytics/tests/test_fixtures.py b/analytics/tests/test_fixtures.py index d1e31142e6bb6..a5e7f47125208 100644 --- a/analytics/tests/test_fixtures.py +++ b/analytics/tests/test_fixtures.py @@ -2,6 +2,7 @@ from analytics.lib.fixtures import generate_time_series_data from zerver.lib.test_classes import ZulipTestCase + # A very light test suite; the code being tested is not run in production. class TestFixtures(ZulipTestCase): def test_deterministic_settings(self) -> None: diff --git a/analytics/tests/test_views.py b/analytics/tests/test_views.py index 1b8d1465aaef5..2ec434b456b2c 100644 --- a/analytics/tests/test_views.py +++ b/analytics/tests/test_views.py @@ -1,24 +1,21 @@ from datetime import datetime, timedelta, timezone from typing import List, Optional - from unittest import mock -from django.http import HttpResponse + import ujson +from django.http import HttpResponse from django.utils.timezone import now as timezone_now from analytics.lib.counts import COUNT_STATS, CountStat from analytics.lib.time_utils import time_range -from analytics.models import FillState, \ - RealmCount, UserCount, last_successful_fill -from analytics.views import rewrite_client_arrays, \ - sort_by_totals, sort_client_labels -from zerver.lib.test_helpers import reset_emails_in_zulip_realm +from analytics.models import FillState, RealmCount, UserCount, last_successful_fill +from analytics.views import rewrite_client_arrays, sort_by_totals, sort_client_labels +from zerver.lib.actions import do_create_multiuse_invite_link, do_send_realm_reactivation_email from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.timestamp import ceiling_to_day, \ - ceiling_to_hour, datetime_to_timestamp -from zerver.lib.actions import do_create_multiuse_invite_link, \ - do_send_realm_reactivation_email -from zerver.models import Client, get_realm, MultiuseInvite +from zerver.lib.test_helpers import reset_emails_in_zulip_realm +from zerver.lib.timestamp import ceiling_to_day, ceiling_to_hour, datetime_to_timestamp +from zerver.models import Client, MultiuseInvite, get_realm + class TestStatsEndpoint(ZulipTestCase): def test_stats(self) -> None: diff --git a/analytics/views.py b/analytics/views.py index 702bb34dc68e8..3444db001e7a5 100644 --- a/analytics/views.py +++ b/analytics/views.py @@ -6,54 +6,75 @@ from collections import defaultdict from datetime import datetime, timedelta, timezone from decimal import Decimal - -from typing import Any, Callable, Dict, List, \ - Optional, Set, Tuple, Type, Union +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union import pytz from django.conf import settings -from django.urls import reverse +from django.core.exceptions import ValidationError +from django.core.validators import URLValidator from django.db import connection from django.db.models.query import QuerySet from django.http import HttpRequest, HttpResponse, HttpResponseNotFound from django.shortcuts import render from django.template import loader +from django.urls import reverse +from django.utils.timesince import timesince from django.utils.timezone import now as timezone_now from django.utils.translation import ugettext as _ -from django.utils.timesince import timesince -from django.core.validators import URLValidator -from django.core.exceptions import ValidationError from jinja2 import Markup as mark_safe -from psycopg2.sql import Composable, Literal, SQL +from psycopg2.sql import SQL, Composable, Literal from analytics.lib.counts import COUNT_STATS, CountStat from analytics.lib.time_utils import time_range -from analytics.models import BaseCount, InstallationCount, \ - RealmCount, StreamCount, UserCount, last_successful_fill, installation_epoch -from confirmation.models import Confirmation, confirmation_url, _properties -from zerver.decorator import require_server_admin, require_server_admin_api, \ - to_utc_datetime, zulip_login_required, require_non_guest_user +from analytics.models import ( + BaseCount, + InstallationCount, + RealmCount, + StreamCount, + UserCount, + installation_epoch, + last_successful_fill, +) +from confirmation.models import Confirmation, _properties, confirmation_url +from confirmation.settings import STATUS_ACTIVE +from zerver.decorator import ( + require_non_guest_user, + require_server_admin, + require_server_admin_api, + to_utc_datetime, + zulip_login_required, +) +from zerver.lib.actions import ( + do_change_plan_type, + do_deactivate_realm, + do_scrub_realm, + do_send_realm_reactivation_email, +) from zerver.lib.exceptions import JsonableError +from zerver.lib.realm_icon import realm_icon_url from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.timestamp import convert_to_UTC, timestamp_to_datetime -from zerver.lib.realm_icon import realm_icon_url -from zerver.views.invite import get_invitee_emails_set from zerver.lib.subdomains import get_subdomain_from_hostname -from zerver.lib.actions import do_change_plan_type, do_deactivate_realm, \ - do_send_realm_reactivation_email, do_scrub_realm +from zerver.lib.timestamp import convert_to_UTC, timestamp_to_datetime from zerver.lib.validator import to_non_negative_int -from confirmation.settings import STATUS_ACTIVE +from zerver.views.invite import get_invitee_emails_set if settings.BILLING_ENABLED: from corporate.lib.stripe import attach_discount_to_realm, get_discount_for_realm -from zerver.models import Client, get_realm, Realm, UserActivity, UserActivityInterval, \ - UserProfile, PreregistrationUser, MultiuseInvite +from zerver.models import ( + Client, + MultiuseInvite, + PreregistrationUser, + Realm, + UserActivity, + UserActivityInterval, + UserProfile, + get_realm, +) if settings.ZILENCER_ENABLED: - from zilencer.models import RemoteInstallationCount, RemoteRealmCount, \ - RemoteZulipServer + from zilencer.models import RemoteInstallationCount, RemoteRealmCount, RemoteZulipServer else: from unittest.mock import Mock RemoteInstallationCount = Mock() # type: ignore[misc] # https://github.com/JukkaL/mypy/issues/1188 diff --git a/confirmation/migrations/0001_initial.py b/confirmation/migrations/0001_initial.py index af6fed33e7f01..75b8fb2f97ffc 100644 --- a/confirmation/migrations/0001_initial.py +++ b/confirmation/migrations/0001_initial.py @@ -1,5 +1,5 @@ -from django.db import models, migrations import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/confirmation/migrations/0002_realmcreationkey.py b/confirmation/migrations/0002_realmcreationkey.py index 2053e28340b83..fac60f8186158 100644 --- a/confirmation/migrations/0002_realmcreationkey.py +++ b/confirmation/migrations/0002_realmcreationkey.py @@ -1,5 +1,5 @@ -from django.db import models, migrations import django.utils.timezone +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/confirmation/migrations/0005_confirmation_realm.py b/confirmation/migrations/0005_confirmation_realm.py index 2dc1088e9d0d1..75179cb797239 100644 --- a/confirmation/migrations/0005_confirmation_realm.py +++ b/confirmation/migrations/0005_confirmation_realm.py @@ -1,7 +1,6 @@ # Generated by Django 1.11.6 on 2017-11-30 00:13 - -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/confirmation/models.py b/confirmation/models.py index c958e57584cf0..8471ed89404c4 100644 --- a/confirmation/models.py +++ b/confirmation/models.py @@ -1,24 +1,23 @@ # Copyright: (c) 2008, Jarek Zgoda __revision__ = '$Id: models.py 28 2009-10-22 15:03:02Z jarek.zgoda $' - import datetime +import string +from random import SystemRandom +from typing import Dict, Optional, Union -from django.db import models -from django.db.models import CASCADE -from django.urls import reverse from django.conf import settings -from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.models import ContentType +from django.db import models +from django.db.models import CASCADE from django.http import HttpRequest, HttpResponse from django.shortcuts import render +from django.urls import reverse from django.utils.timezone import now as timezone_now -from zerver.models import PreregistrationUser, EmailChangeStatus, MultiuseInvite, \ - UserProfile, Realm -from random import SystemRandom -import string -from typing import Dict, Optional, Union +from zerver.models import EmailChangeStatus, MultiuseInvite, PreregistrationUser, Realm, UserProfile + class ConfirmationKeyException(Exception): WRONG_LENGTH = 1 diff --git a/corporate/lib/stripe.py b/corporate/lib/stripe.py index 54ee6d3610710..2eabbcabc343e 100644 --- a/corporate/lib/stripe.py +++ b/corporate/lib/stripe.py @@ -1,26 +1,31 @@ -from datetime import datetime, timedelta -from decimal import Decimal -from functools import wraps import logging import math import os -from typing import Any, Callable, Dict, Optional, TypeVar, Tuple, cast -import ujson +from datetime import datetime, timedelta +from decimal import Decimal +from functools import wraps +from typing import Any, Callable, Dict, Optional, Tuple, TypeVar, cast +import stripe +import ujson from django.conf import settings +from django.core.signing import Signer from django.db import transaction -from django.utils.translation import ugettext as _ from django.utils.timezone import now as timezone_now -from django.core.signing import Signer -import stripe +from django.utils.translation import ugettext as _ +from corporate.models import ( + Customer, + CustomerPlan, + LicenseLedger, + get_current_plan_by_customer, + get_current_plan_by_realm, + get_customer_by_realm, +) from zerver.lib.logging_util import log_to_file from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime from zerver.lib.utils import generate_random_token -from zerver.models import Realm, UserProfile, RealmAuditLog -from corporate.models import Customer, CustomerPlan, LicenseLedger, \ - get_current_plan_by_customer, get_customer_by_realm, \ - get_current_plan_by_realm +from zerver.models import Realm, RealmAuditLog, UserProfile from zproject.config import get_secret STRIPE_PUBLISHABLE_KEY = get_secret('stripe_publishable_key') diff --git a/corporate/models.py b/corporate/models.py index bb6ea75fe7437..715f10d7e3c34 100644 --- a/corporate/models.py +++ b/corporate/models.py @@ -7,6 +7,7 @@ from zerver.models import Realm + class Customer(models.Model): realm: Realm = models.OneToOneField(Realm, on_delete=CASCADE) stripe_customer_id: str = models.CharField(max_length=255, null=True, unique=True) diff --git a/corporate/tests/test_stripe.py b/corporate/tests/test_stripe.py index 1be9d5be65b4f..cfd9586917504 100644 --- a/corporate/tests/test_stripe.py +++ b/corporate/tests/test_stripe.py @@ -1,43 +1,66 @@ -from datetime import datetime, timedelta, timezone -from decimal import Decimal -from functools import wraps -from unittest.mock import Mock, patch +import json import operator import os import re import sys -from typing import Any, Callable, Dict, List, Optional, TypeVar, Tuple, cast -import ujson -import json -import responses +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from functools import wraps +from typing import Any, Callable, Dict, List, Optional, Tuple, TypeVar, cast +from unittest.mock import Mock, patch +import responses +import stripe +import ujson +from django.conf import settings from django.core import signing -from django.urls.resolvers import get_resolver from django.http import HttpResponse -from django.conf import settings +from django.urls.resolvers import get_resolver from django.utils.timezone import now as timezone_now -import stripe - -from zerver.lib.actions import do_deactivate_user, do_create_user, \ - do_activate_user, do_reactivate_user, do_deactivate_realm, \ - do_reactivate_realm +from corporate.lib.stripe import ( + MAX_INVOICED_LICENSES, + MIN_INVOICED_LICENSES, + BillingError, + StripeCardError, + add_months, + attach_discount_to_realm, + catch_stripe_errors, + compute_plan_parameters, + get_discount_for_realm, + get_latest_seat_count, + invoice_plan, + invoice_plans_as_needed, + make_end_of_cycle_updates_if_needed, + next_month, + process_initial_upgrade, + sign_string, + stripe_get_customer, + unsign_string, + update_license_ledger_for_automanaged_plan, + update_license_ledger_if_needed, + update_or_create_stripe_customer, +) +from corporate.models import ( + Customer, + CustomerPlan, + LicenseLedger, + get_current_plan_by_customer, + get_current_plan_by_realm, + get_customer_by_realm, +) +from zerver.lib.actions import ( + do_activate_user, + do_create_user, + do_deactivate_realm, + do_deactivate_user, + do_reactivate_realm, + do_reactivate_user, +) from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import reset_emails_in_zulip_realm -from zerver.lib.timestamp import timestamp_to_datetime, datetime_to_timestamp -from zerver.models import Realm, UserProfile, get_realm, RealmAuditLog -from corporate.lib.stripe import catch_stripe_errors, attach_discount_to_realm, \ - get_latest_seat_count, sign_string, unsign_string, \ - BillingError, StripeCardError, stripe_get_customer, \ - MIN_INVOICED_LICENSES, MAX_INVOICED_LICENSES, \ - add_months, next_month, \ - compute_plan_parameters, update_or_create_stripe_customer, \ - process_initial_upgrade, make_end_of_cycle_updates_if_needed, \ - update_license_ledger_if_needed, update_license_ledger_for_automanaged_plan, \ - invoice_plan, invoice_plans_as_needed, get_discount_for_realm -from corporate.models import Customer, CustomerPlan, LicenseLedger, \ - get_customer_by_realm, get_current_plan_by_customer, \ - get_current_plan_by_realm +from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime +from zerver.models import Realm, RealmAuditLog, UserProfile, get_realm CallableT = TypeVar('CallableT', bound=Callable[..., Any]) diff --git a/corporate/urls.py b/corporate/urls.py index c799f7edf2746..4ebb7c2e6b92f 100644 --- a/corporate/urls.py +++ b/corporate/urls.py @@ -1,8 +1,8 @@ from typing import Any -from django.views.generic import TemplateView from django.conf.urls import include from django.urls import path +from django.views.generic import TemplateView import corporate.views from zerver.lib.rest import rest_dispatch diff --git a/corporate/views.py b/corporate/views.py index 0f8c8f3ffc9fe..57f6e8d33b6fa 100644 --- a/corporate/views.py +++ b/corporate/views.py @@ -1,30 +1,45 @@ import logging from decimal import Decimal -import stripe -from typing import Any, Dict, cast, Optional, Union +from typing import Any, Dict, Optional, Union, cast +import stripe +from django.conf import settings from django.core import signing from django.http import HttpRequest, HttpResponse, HttpResponseRedirect -from django.utils.timezone import now as timezone_now -from django.utils.translation import ugettext as _ from django.shortcuts import render from django.urls import reverse -from django.conf import settings +from django.utils.timezone import now as timezone_now +from django.utils.translation import ugettext as _ -from zerver.decorator import zulip_login_required, require_billing_access +from corporate.lib.stripe import ( + DEFAULT_INVOICE_DAYS_UNTIL_DUE, + MAX_INVOICED_LICENSES, + MIN_INVOICED_LICENSES, + STRIPE_PUBLISHABLE_KEY, + BillingError, + do_change_plan_status, + do_replace_payment_source, + downgrade_now, + get_latest_seat_count, + make_end_of_cycle_updates_if_needed, + process_initial_upgrade, + renewal_amount, + sign_string, + start_of_next_billing_cycle, + stripe_get_customer, + unsign_string, +) +from corporate.models import ( + CustomerPlan, + get_current_plan_by_customer, + get_current_plan_by_realm, + get_customer_by_realm, +) +from zerver.decorator import require_billing_access, zulip_login_required from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_error, json_success -from zerver.lib.validator import check_string, check_int +from zerver.lib.validator import check_int, check_string from zerver.models import UserProfile -from corporate.lib.stripe import STRIPE_PUBLISHABLE_KEY, \ - stripe_get_customer, get_latest_seat_count, \ - process_initial_upgrade, sign_string, \ - unsign_string, BillingError, do_change_plan_status, do_replace_payment_source, \ - MIN_INVOICED_LICENSES, MAX_INVOICED_LICENSES, DEFAULT_INVOICE_DAYS_UNTIL_DUE, \ - start_of_next_billing_cycle, renewal_amount, \ - make_end_of_cycle_updates_if_needed, downgrade_now -from corporate.models import CustomerPlan, get_current_plan_by_customer, \ - get_customer_by_realm, get_current_plan_by_realm billing_logger = logging.getLogger('corporate.stripe') diff --git a/docs/conf.py b/docs/conf.py index f11bad4cc1fbf..45f49e78bda71 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,10 +10,8 @@ # # All configuration values have a default; values that are commented out # serve to show the default. - import os import sys - from typing import Any, Dict, List, Optional # If extensions (or modules to document with autodoc) are in another directory, @@ -310,6 +308,7 @@ # See https://github.com/zulip/zulip/issues/13263 for details. from recommonmark.parser import CommonMarkParser + class CustomCommonMarkParser(CommonMarkParser): def visit_document(self, node): pass diff --git a/frontend_tests/run-casper b/frontend_tests/run-casper index e59ad4f3b6d1d..0261bdf8c4521 100755 --- a/frontend_tests/run-casper +++ b/frontend_tests/run-casper @@ -1,10 +1,10 @@ #!/usr/bin/env python3 import argparse -import subprocess -import sys -import os import glob +import os import shlex +import subprocess +import sys # # In order to use remote casperjs debugging, pass the --remote-debug flag @@ -64,13 +64,14 @@ sys.path.insert(0, ZULIP_PATH) # check for the venv from tools.lib import sanity_check + sanity_check.check_venv(__file__) +from typing import Iterable, List + from tools.lib.test_script import assert_provisioning_status_ok, find_js_test_files from tools.lib.test_server import test_server_running -from typing import Iterable, List - assert_provisioning_status_ok(options.force) os.chdir(ZULIP_PATH) diff --git a/manage.py b/manage.py index ce8204c393070..30f8fb0e7b58a 100755 --- a/manage.py +++ b/manage.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 +import configparser import os import sys -import configparser + if sys.version_info <= (3, 0): print("Error: Zulip is a Python 3 project, and cannot be run with Python 2.") print("Use e.g. `/path/to/manage.py` not `python /path/to/manage.py`.") @@ -37,6 +38,7 @@ from django.conf import settings from django.core.management import execute_from_command_line from django.core.management.base import CommandError + from scripts.lib.zulip_tools import log_management_command log_management_command(" ".join(sys.argv), settings.MANAGEMENT_LOG_PATH) diff --git a/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_cron_file b/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_cron_file index 64f53c11e78cb..86df41348aee0 100755 --- a/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_cron_file +++ b/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_cron_file @@ -6,6 +6,7 @@ import sys import time from typing import Tuple + def nagios_from_file(results_file: str, max_time_diff: int=60 * 2) -> 'Tuple[int, str]': """Returns a nagios-appropriate string and return code obtained by parsing the desired file on disk. The file on disk should be of format diff --git a/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_send_receive_time b/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_send_receive_time index 739a78c7a4243..67347bfa456f4 100755 --- a/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_send_receive_time +++ b/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_send_receive_time @@ -8,12 +8,12 @@ It supports both munin and nagios outputs It must be run on a machine that is using the live database for the Django ORM. """ -import sys import argparse +import os import random +import sys import time import traceback -import os sys.path.append('.') sys.path.append('/home/zulip/deployments/current') @@ -21,10 +21,10 @@ from scripts.lib.setup_path import setup_path setup_path() -import django - from typing import Any, Dict, List, Optional +import django + usage = """Usage: send-receive.py [options] [config] 'config' is optional, if present will return config info. @@ -75,9 +75,10 @@ os.environ['DJANGO_SETTINGS_MODULE'] = "zproject.settings" django.setup() -from zerver.models import get_system_bot from django.conf import settings +from zerver.models import get_system_bot + states = { "OK": 0, "WARNING": 1, diff --git a/puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_postgres_replication_lag b/puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_postgres_replication_lag index 32dd089e0514d..f209cc32e1ef1 100755 --- a/puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_postgres_replication_lag +++ b/puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_postgres_replication_lag @@ -4,10 +4,9 @@ Nagios plugin to check the difference between the primary and secondary Postgres servers' xlog location. """ - import configparser -import subprocess import re +import subprocess from typing import NoReturn states = { diff --git a/puppet/zulip/files/nagios_plugins/zulip_postgres_appdb/check_fts_update_log b/puppet/zulip/files/nagios_plugins/zulip_postgres_appdb/check_fts_update_log index 9b19f5cbc693f..d5665b98e158d 100755 --- a/puppet/zulip/files/nagios_plugins/zulip_postgres_appdb/check_fts_update_log +++ b/puppet/zulip/files/nagios_plugins/zulip_postgres_appdb/check_fts_update_log @@ -4,6 +4,7 @@ Nagios plugin to check the length of the FTS update log. """ import sys + sys.path.append('/home/zulip/deployments/current') try: from scripts.lib.setup_path import setup_path diff --git a/puppet/zulip/files/postgresql/pg_backup_and_purge b/puppet/zulip/files/postgresql/pg_backup_and_purge index e4cd6d3d39833..96c1fc59793a7 100755 --- a/puppet/zulip/files/postgresql/pg_backup_and_purge +++ b/puppet/zulip/files/postgresql/pg_backup_and_purge @@ -1,16 +1,16 @@ #!/usr/bin/env python3 - -import os -import sys import glob +import logging +import os import shlex import subprocess -import logging -import dateutil.parser +import sys import time from datetime import datetime, timedelta, timezone from typing import Dict, List +import dateutil.parser + logging.Formatter.converter = time.gmtime logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s") logger = logging.getLogger(__name__) diff --git a/puppet/zulip/files/postgresql/process_fts_updates b/puppet/zulip/files/postgresql/process_fts_updates index 8aedfe20ffedc..d79a6bcc1e467 100755 --- a/puppet/zulip/files/postgresql/process_fts_updates +++ b/puppet/zulip/files/postgresql/process_fts_updates @@ -23,14 +23,15 @@ except ImportError: pass import argparse -import psycopg2 -import psycopg2.extensions -import select -import time -import logging import configparser -import sys +import logging import os +import select +import sys +import time + +import psycopg2 +import psycopg2.extensions BATCH_SIZE = 1000 diff --git a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_personal_zephyr_mirrors b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_personal_zephyr_mirrors index 7e66611cbf0f8..fe498f3e0458a 100755 --- a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_personal_zephyr_mirrors +++ b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_personal_zephyr_mirrors @@ -8,9 +8,9 @@ This script works by just monitoring the files under mirrors when they receive the messages sent every minute by /etc/cron.d/test_zephyr_personal_mirrors """ -from typing import Dict import os import time +from typing import Dict RESULTS_DIR: str = "/home/zulip/mirror_status" diff --git a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness index 13b1fdff8afa0..59c4cd4303452 100755 --- a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness +++ b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness @@ -24,9 +24,9 @@ sys.path.append('/home/zulip/deployments/current/zerver') django.setup() -from zerver.models import UserActivity +from typing import Any, Dict, Optional, Set -from typing import Any, Dict, Set, Optional +from zerver.models import UserActivity states: Dict[str, int] = { "OK": 0, diff --git a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_zephyr_mirror b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_zephyr_mirror index ed7d8a9ed5260..d095b257ad73e 100755 --- a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_zephyr_mirror +++ b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_zephyr_mirror @@ -9,9 +9,9 @@ run out of cron. See puppet/zulip_ops/files/cron.d/zephyr-mirror for the crontab details. """ -from typing import Dict import os import time +from typing import Dict RESULTS_FILE = "/var/lib/nagios_state/check-mirroring-results" diff --git a/puppet/zulip_ops/files/zulip-ec2-configure-interfaces b/puppet/zulip_ops/files/zulip-ec2-configure-interfaces index d5ecbcf852151..9c94333e94972 100755 --- a/puppet/zulip_ops/files/zulip-ec2-configure-interfaces +++ b/puppet/zulip_ops/files/zulip-ec2-configure-interfaces @@ -40,16 +40,15 @@ Note that it currently does not handle the deconfiguration of interfaces. ''' - -import sys import logging import logging.handlers import subprocess +import sys +from typing import Optional import boto.utils import netifaces -from typing import Optional def address_of(device_id: int) -> Optional[str]: try: diff --git a/scripts/lib/check_rabbitmq_queue.py b/scripts/lib/check_rabbitmq_queue.py index cb4095539d1ee..614728aa8eb44 100644 --- a/scripts/lib/check_rabbitmq_queue.py +++ b/scripts/lib/check_rabbitmq_queue.py @@ -1,9 +1,8 @@ +import json import os import re -import time import subprocess -import json - +import time from collections import defaultdict from typing import Any, DefaultDict, Dict, List diff --git a/scripts/lib/clean-unused-caches b/scripts/lib/clean-unused-caches index 480c6a2e034cd..5c548918bc788 100755 --- a/scripts/lib/clean-unused-caches +++ b/scripts/lib/clean-unused-caches @@ -4,8 +4,9 @@ import sys ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) +from scripts.lib import clean_emoji_cache, clean_node_cache, clean_venv_cache from scripts.lib.zulip_tools import parse_cache_script_args -from scripts.lib import clean_venv_cache, clean_node_cache, clean_emoji_cache + def main() -> None: args = parse_cache_script_args("This script cleans unused zulip caches.") diff --git a/scripts/lib/clean_emoji_cache.py b/scripts/lib/clean_emoji_cache.py index aaa6936437b04..6457949c54064 100755 --- a/scripts/lib/clean_emoji_cache.py +++ b/scripts/lib/clean_emoji_cache.py @@ -2,14 +2,16 @@ import argparse import os import sys - from typing import Set ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import \ - get_environment, get_recent_deployments, \ - parse_cache_script_args, purge_unused_caches +from scripts.lib.zulip_tools import ( + get_environment, + get_recent_deployments, + parse_cache_script_args, + purge_unused_caches, +) ENV = get_environment() EMOJI_CACHE_PATH = "/srv/zulip-emoji-cache" diff --git a/scripts/lib/clean_node_cache.py b/scripts/lib/clean_node_cache.py index f4b2868738654..b1fd1db0c3c42 100755 --- a/scripts/lib/clean_node_cache.py +++ b/scripts/lib/clean_node_cache.py @@ -2,14 +2,16 @@ import argparse import os import sys - from typing import Set ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import \ - get_environment, get_recent_deployments, parse_cache_script_args, \ - purge_unused_caches +from scripts.lib.zulip_tools import ( + get_environment, + get_recent_deployments, + parse_cache_script_args, + purge_unused_caches, +) ENV = get_environment() NODE_MODULES_CACHE_PATH = "/srv/zulip-npm-cache" diff --git a/scripts/lib/clean_venv_cache.py b/scripts/lib/clean_venv_cache.py index dd03b8aa99ecf..513b4d532f4fe 100755 --- a/scripts/lib/clean_venv_cache.py +++ b/scripts/lib/clean_venv_cache.py @@ -3,15 +3,17 @@ import glob import os import sys - from typing import Set ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) from scripts.lib.hash_reqs import expand_reqs, hash_deps -from scripts.lib.zulip_tools import \ - get_environment, get_recent_deployments, parse_cache_script_args, \ - purge_unused_caches +from scripts.lib.zulip_tools import ( + get_environment, + get_recent_deployments, + parse_cache_script_args, + purge_unused_caches, +) ENV = get_environment() VENV_CACHE_DIR = '/srv/zulip-venv-cache' diff --git a/scripts/lib/create-production-venv b/scripts/lib/create-production-venv index d7b50cd76bb2d..16c27a4a13eac 100755 --- a/scripts/lib/create-production-venv +++ b/scripts/lib/create-production-venv @@ -1,17 +1,14 @@ #!/usr/bin/env python3 - -import os import argparse +import os import sys ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) if ZULIP_PATH not in sys.path: sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import os_families, overwrite_symlink, run, parse_os_release -from scripts.lib.setup_venv import ( - setup_virtualenv, get_venv_dependencies, -) +from scripts.lib.setup_venv import get_venv_dependencies, setup_virtualenv +from scripts.lib.zulip_tools import os_families, overwrite_symlink, parse_os_release, run parser = argparse.ArgumentParser(description="Create a production virtualenv with caching") parser.add_argument("deploy_path") diff --git a/scripts/lib/create-thumbor-venv b/scripts/lib/create-thumbor-venv index 3ecf1bf53392f..f22a8c7219e72 100755 --- a/scripts/lib/create-thumbor-venv +++ b/scripts/lib/create-thumbor-venv @@ -1,17 +1,18 @@ #!/usr/bin/env python3 - -import os import argparse +import os import sys ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) if ZULIP_PATH not in sys.path: sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import os_families, run, parse_os_release from scripts.lib.setup_venv import ( - setup_virtualenv, THUMBOR_VENV_DEPENDENCIES, YUM_THUMBOR_VENV_DEPENDENCIES, + THUMBOR_VENV_DEPENDENCIES, + YUM_THUMBOR_VENV_DEPENDENCIES, + setup_virtualenv, ) +from scripts.lib.zulip_tools import os_families, parse_os_release, run parser = argparse.ArgumentParser(description="Create a thumbor virtualenv with caching") parser.add_argument("deploy_path") diff --git a/scripts/lib/email-mirror-postfix b/scripts/lib/email-mirror-postfix index e2cd3d58e0447..7b793cb7ae312 100755 --- a/scripts/lib/email-mirror-postfix +++ b/scripts/lib/email-mirror-postfix @@ -40,21 +40,16 @@ Also you can use optional keys to configure the script and change default values -t Disable sending request to the Zulip server. Default value: False. """ - +import argparse +import json import os +import posix import ssl import sys - -import argparse - -import posix -import json - -from urllib.parse import urljoin, urlencode -from urllib.request import Request, urlopen -from urllib.error import HTTPError from configparser import RawConfigParser - +from urllib.error import HTTPError +from urllib.parse import urlencode, urljoin +from urllib.request import Request, urlopen parser = argparse.ArgumentParser() diff --git a/scripts/lib/hash_reqs.py b/scripts/lib/hash_reqs.py index 6115b5ef12d14..219ee204459f7 100755 --- a/scripts/lib/hash_reqs.py +++ b/scripts/lib/hash_reqs.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 -import os -import sys import argparse import hashlib +import os +import sys from typing import Iterable, List, MutableSet + def expand_reqs_helper(fpath: str, visited: MutableSet[str]) -> List[str]: if fpath in visited: return [] diff --git a/scripts/lib/node_cache.py b/scripts/lib/node_cache.py index 42181daab2e89..b292e8c3af161 100644 --- a/scripts/lib/node_cache.py +++ b/scripts/lib/node_cache.py @@ -1,10 +1,10 @@ -import os import hashlib import json +import os import shutil +from typing import List, Optional -from typing import Optional, List -from scripts.lib.zulip_tools import subprocess_text_output, run +from scripts.lib.zulip_tools import run, subprocess_text_output ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ZULIP_SRV_PATH = "/srv" diff --git a/scripts/lib/pythonrc.py b/scripts/lib/pythonrc.py index 2b5e4a9a16a4b..016d180943f31 100644 --- a/scripts/lib/pythonrc.py +++ b/scripts/lib/pythonrc.py @@ -1,8 +1,9 @@ try: from django.conf import settings # noqa: F401 - from zerver.models import * # noqa: F401, F403 - from zerver.lib.actions import * # noqa: F401, F403 + from analytics.models import * # noqa: F401, F403 + from zerver.lib.actions import * # noqa: F401, F403 + from zerver.models import * # noqa: F401, F403 except Exception: import traceback print("\nException importing Zulip core modules on startup!") diff --git a/scripts/lib/queue_workers.py b/scripts/lib/queue_workers.py index 12de3719929e7..68b337c38a1f4 100755 --- a/scripts/lib/queue_workers.py +++ b/scripts/lib/queue_workers.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import argparse import os import sys @@ -13,6 +12,7 @@ os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.settings' import django + django.setup() from zerver.worker.queue_processors import get_active_worker_queues diff --git a/scripts/lib/setup_path.py b/scripts/lib/setup_path.py index c8748bd9daf28..20a190242aa2c 100644 --- a/scripts/lib/setup_path.py +++ b/scripts/lib/setup_path.py @@ -1,10 +1,10 @@ """ Use libraries from a virtualenv (by modifying sys.path) in production. """ - import os import sys + def setup_path() -> None: if os.path.basename(sys.prefix) != "zulip-py3-venv": BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/scripts/lib/setup_venv.py b/scripts/lib/setup_venv.py index b3c696806fb93..155df9818ec2c 100644 --- a/scripts/lib/setup_venv.py +++ b/scripts/lib/setup_venv.py @@ -2,10 +2,10 @@ import os import shutil import subprocess -from scripts.lib.zulip_tools import run, run_as_root, ENDC, WARNING, os_families -from scripts.lib.hash_reqs import expand_reqs +from typing import List, Optional, Set, Tuple -from typing import List, Optional, Tuple, Set +from scripts.lib.hash_reqs import expand_reqs +from scripts.lib.zulip_tools import ENDC, WARNING, os_families, run, run_as_root ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) VENV_CACHE_PATH = "/srv/zulip-venv-cache" diff --git a/scripts/lib/sharding.py b/scripts/lib/sharding.py index df68fbfe0958a..fad230cca06da 100755 --- a/scripts/lib/sharding.py +++ b/scripts/lib/sharding.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import json import os import subprocess @@ -14,6 +13,7 @@ from scripts.lib.zulip_tools import get_config_file + def write_realm_nginx_config_line(f: Any, host: str, port: str) -> None: f.write("""if ($host = '{}') {{ set $tornado_server http://tornado{}; diff --git a/scripts/lib/unpack-zulip b/scripts/lib/unpack-zulip index 4d73e1b257259..bf22d0205308f 100755 --- a/scripts/lib/unpack-zulip +++ b/scripts/lib/unpack-zulip @@ -1,17 +1,24 @@ #!/usr/bin/env python3 +import glob import os import shutil -import sys import subprocess +import sys import tempfile -import glob os.environ["PYTHONUNBUFFERED"] = "y" sys.path.append(os.path.join(os.path.dirname(__file__), '..', "..")) -from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, FAIL, ENDC, make_deploy_path, \ - get_deployment_version, is_invalid_upgrade, overwrite_symlink import version +from scripts.lib.zulip_tools import ( + DEPLOYMENTS_DIR, + ENDC, + FAIL, + get_deployment_version, + is_invalid_upgrade, + make_deploy_path, + overwrite_symlink, +) if len(sys.argv) != 2: print(FAIL + f"Usage: {sys.argv[0]} " + ENDC) diff --git a/scripts/lib/upgrade-zulip b/scripts/lib/upgrade-zulip index 1454b4a0b085c..5a1105ea19229 100755 --- a/scripts/lib/upgrade-zulip +++ b/scripts/lib/upgrade-zulip @@ -1,19 +1,27 @@ #!/usr/bin/env python3 +import configparser +import logging import os import shutil -import sys import subprocess -import logging +import sys import time -import configparser TARBALL_ARCHIVE_PATH = "/home/zulip/archives" os.environ["PYTHONUNBUFFERED"] = "y" sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..')) -from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, FAIL, ENDC, \ - su_to_zulip, get_deployment_lock, release_deployment_lock, assert_running_as_root, \ - get_config_file, get_deploy_options +from scripts.lib.zulip_tools import ( + DEPLOYMENTS_DIR, + ENDC, + FAIL, + assert_running_as_root, + get_config_file, + get_deploy_options, + get_deployment_lock, + release_deployment_lock, + su_to_zulip, +) config_file: configparser.RawConfigParser = get_config_file() deploy_options = get_deploy_options(config_file) diff --git a/scripts/lib/upgrade-zulip-from-git b/scripts/lib/upgrade-zulip-from-git index 406354b14b245..2359bf9f2c505 100755 --- a/scripts/lib/upgrade-zulip-from-git +++ b/scripts/lib/upgrade-zulip-from-git @@ -1,18 +1,27 @@ #!/usr/bin/env python3 +import argparse +import logging import os -import sys import subprocess -import logging +import sys import time -import argparse LOCAL_GIT_CACHE_DIR = '/srv/zulip.git' os.environ["PYTHONUNBUFFERED"] = "y" sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..')) -from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, make_deploy_path, \ - get_deployment_lock, overwrite_symlink, release_deployment_lock, su_to_zulip, assert_running_as_root, \ - get_config_file, get_deploy_options, get_config +from scripts.lib.zulip_tools import ( + DEPLOYMENTS_DIR, + assert_running_as_root, + get_config, + get_config_file, + get_deploy_options, + get_deployment_lock, + make_deploy_path, + overwrite_symlink, + release_deployment_lock, + su_to_zulip, +) config_file = get_config_file() deploy_options = get_deploy_options(config_file) diff --git a/scripts/lib/upgrade-zulip-stage-2 b/scripts/lib/upgrade-zulip-stage-2 index 8cbc48d73008c..3677f3a61c39e 100755 --- a/scripts/lib/upgrade-zulip-stage-2 +++ b/scripts/lib/upgrade-zulip-stage-2 @@ -8,10 +8,10 @@ import argparse import configparser import glob import hashlib -import subprocess +import logging import os +import subprocess import sys -import logging import time os.environ["PYTHONUNBUFFERED"] = "y" @@ -22,8 +22,12 @@ os.environ["LANG"] = "en_US.UTF-8" os.environ["LANGUAGE"] = "en_US.UTF-8" sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..')) -from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, su_to_zulip, \ - assert_running_as_root, parse_os_release +from scripts.lib.zulip_tools import ( + DEPLOYMENTS_DIR, + assert_running_as_root, + parse_os_release, + su_to_zulip, +) assert_running_as_root() diff --git a/scripts/lib/zulip_tools.py b/scripts/lib/zulip_tools.py index dce756d64c901..39e96956f770a 100755 --- a/scripts/lib/zulip_tools.py +++ b/scripts/lib/zulip_tools.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 import argparse +import configparser import datetime import functools import hashlib +import json import logging import os import pwd @@ -13,11 +15,8 @@ import sys import tempfile import time -import json import uuid -import configparser - -from typing import Sequence, Set, Any, Dict, List +from typing import Any, Dict, List, Sequence, Set DEPLOYMENTS_DIR = "/home/zulip/deployments" LOCK_DIR = os.path.join(DEPLOYMENTS_DIR, "lock") diff --git a/scripts/nagios/check-rabbitmq-consumers b/scripts/nagios/check-rabbitmq-consumers index cf4bf7e0cad65..8cf11a9f2f16b 100755 --- a/scripts/nagios/check-rabbitmq-consumers +++ b/scripts/nagios/check-rabbitmq-consumers @@ -1,12 +1,11 @@ #!/usr/bin/env python3 - -import sys -import time import argparse import configparser -from collections import defaultdict import os import subprocess +import sys +import time +from collections import defaultdict from typing import Dict ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/scripts/nagios/cron_file_helper.py b/scripts/nagios/cron_file_helper.py index f8a7954e3cdb3..ca92ece168e6b 100644 --- a/scripts/nagios/cron_file_helper.py +++ b/scripts/nagios/cron_file_helper.py @@ -1,7 +1,7 @@ import time - from typing import Tuple + def nagios_from_file(results_file: str) -> Tuple[int, str]: """Returns a nagios-appropriate string and return code obtained by parsing the desired file on disk. The file on disk should be of format diff --git a/scripts/purge-old-deployments b/scripts/purge-old-deployments index f82759c94b95d..a1fcc56df76ba 100755 --- a/scripts/purge-old-deployments +++ b/scripts/purge-old-deployments @@ -3,13 +3,12 @@ import argparse import os import subprocess import sys - from typing import Set ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, get_recent_deployments, \ - may_be_perform_purging +from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, get_recent_deployments, may_be_perform_purging + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( diff --git a/scripts/restart-server b/scripts/restart-server index 6a0bb789e5445..33b2f132c5b75 100755 --- a/scripts/restart-server +++ b/scripts/restart-server @@ -1,16 +1,16 @@ #!/usr/bin/env python3 import argparse import configparser +import logging import os -import sys import pwd +import shlex import subprocess -import logging +import sys import time -import shlex sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -from scripts.lib.zulip_tools import ENDC, OKGREEN, WARNING, DEPLOYMENTS_DIR, overwrite_symlink +from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, ENDC, OKGREEN, WARNING, overwrite_symlink logging.Formatter.converter = time.gmtime logging.basicConfig(format="%(asctime)s restart-server: %(message)s", diff --git a/scripts/setup/flush-memcached b/scripts/setup/flush-memcached index d9c59ddd4cced..dafc37bc6bd81 100755 --- a/scripts/setup/flush-memcached +++ b/scripts/setup/flush-memcached @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import os import sys @@ -10,9 +9,10 @@ from scripts.lib.setup_path import setup_path setup_path() -from zproject import settings import pylibmc +from zproject import settings + pylibmc.Client( [settings.MEMCACHED_LOCATION], binary=True, diff --git a/scripts/setup/generate_secrets.py b/scripts/setup/generate_secrets.py index 864c684ec7044..1bb880e2786d2 100755 --- a/scripts/setup/generate_secrets.py +++ b/scripts/setup/generate_secrets.py @@ -1,9 +1,7 @@ #!/usr/bin/env python3 # This tools generates /etc/zulip/zulip-secrets.conf - -import sys import os - +import sys from typing import Dict, List BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -15,8 +13,8 @@ os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.settings' import argparse -import uuid import configparser +import uuid os.chdir(os.path.join(os.path.dirname(__file__), '..', '..')) @@ -140,6 +138,7 @@ def add_secret(name: str, value: str) -> None: # file directly. import redis + from zerver.lib.redis_utils import get_redis_client redis_password = random_token() diff --git a/scripts/setup/inline_email_css.py b/scripts/setup/inline_email_css.py index 58bf6bdc3eb71..418ff150f454f 100755 --- a/scripts/setup/inline_email_css.py +++ b/scripts/setup/inline_email_css.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 import os +from typing import Set -from premailer import Premailer from cssutils import profile -from cssutils.profiles import Profiles, properties, macros -from typing import Set +from cssutils.profiles import Profiles, macros, properties +from premailer import Premailer ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../') EMAIL_TEMPLATES_PATH = os.path.join(ZULIP_PATH, 'templates', 'zerver', 'emails') diff --git a/scripts/setup/restore-backup b/scripts/setup/restore-backup index 00f3b3e7131ec..432cf4a61c758 100755 --- a/scripts/setup/restore-backup +++ b/scripts/setup/restore-backup @@ -1,17 +1,15 @@ #!/usr/bin/env python3 - import argparse import os import re import subprocess import sys import tempfile - from typing import IO BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(BASE_DIR) -from scripts.lib.zulip_tools import su_to_zulip, run +from scripts.lib.zulip_tools import run, su_to_zulip POSTGRES_USER = "postgres" diff --git a/scripts/zulip-puppet-apply b/scripts/zulip-puppet-apply index 672591c44a42f..c0e1194c70b63 100755 --- a/scripts/zulip-puppet-apply +++ b/scripts/zulip-puppet-apply @@ -1,11 +1,11 @@ #!/usr/bin/env python3 - -import os -import sys -import subprocess import configparser +import os import re -from lib.zulip_tools import parse_os_release, assert_running_as_root +import subprocess +import sys + +from lib.zulip_tools import assert_running_as_root, parse_os_release assert_running_as_root() BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) diff --git a/static/assets/favicon/generate b/static/assets/favicon/generate index 9ba478fd64cf4..ae94f76001588 100755 --- a/static/assets/favicon/generate +++ b/static/assets/favicon/generate @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import xml.etree.ElementTree as ET import subprocess +from xml.etree import ElementTree as ET # Generates the favicon images containing unread message counts. diff --git a/tools/check-capitalization b/tools/check-capitalization index c99f08ceab567..1645ed98161dc 100755 --- a/tools/check-capitalization +++ b/tools/check-capitalization @@ -2,6 +2,7 @@ # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) import argparse @@ -12,8 +13,7 @@ import subprocess import sys sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -from scripts.lib.zulip_tools import WARNING, FAIL, ENDC - +from scripts.lib.zulip_tools import ENDC, FAIL, WARNING from tools.lib.capitalization import check_capitalization DJANGO_PO_REGEX = re.compile('msgid "(.*?)"') diff --git a/tools/check-frontend-i18n b/tools/check-frontend-i18n index 24c40a70da4b6..6c119cd201a24 100755 --- a/tools/check-frontend-i18n +++ b/tools/check-frontend-i18n @@ -1,18 +1,20 @@ #!/usr/bin/env python3 - from typing import List + # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) import argparse import json import os -import sys import subprocess +import sys sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -from scripts.lib.zulip_tools import WARNING, FAIL, ENDC +from scripts.lib.zulip_tools import ENDC, FAIL, WARNING + def find_handlebars(translatable_strings: List[str]) -> List[str]: errored = [] diff --git a/tools/check-issue-labels b/tools/check-issue-labels index fea01f1fc79ee..5115df2fe6f6a 100755 --- a/tools/check-issue-labels +++ b/tools/check-issue-labels @@ -1,13 +1,13 @@ #!/usr/bin/env python3 - -import requests -import re import argparse -import sys import os +import re +import sys +from typing import Any, Dict, Optional + +import requests import ConfigParser -from typing import Any, Dict, Optional # Scans zulip repositary for issues that don't have any `area` labels. # GitHub API token is required as GitHub limits unauthenticated diff --git a/tools/check-provision b/tools/check-provision index 41d5a5bcbb9b3..fe64e18daa0c2 100755 --- a/tools/check-provision +++ b/tools/check-provision @@ -1,16 +1,14 @@ #!/usr/bin/env python3 - -import os import argparse +import os import sys tools_dir = os.path.dirname(os.path.abspath(__file__)) root_dir = os.path.dirname(tools_dir) sys.path.insert(0, root_dir) -from tools.lib.test_script import ( - assert_provisioning_status_ok, -) +from tools.lib.test_script import assert_provisioning_status_ok + def run() -> None: parser = argparse.ArgumentParser() diff --git a/tools/check-templates b/tools/check-templates index 7d1859cbb4533..deb796482a996 100755 --- a/tools/check-templates +++ b/tools/check-templates @@ -1,18 +1,20 @@ #!/usr/bin/env python3 -from lib.template_parser import validate -from lib.html_branches import build_id_dict -from lib.pretty_print import validate_indent_html import argparse -import sys import logging +import sys # check for the venv from lib import sanity_check +from lib.html_branches import build_id_dict +from lib.pretty_print import validate_indent_html +from lib.template_parser import validate + sanity_check.check_venv(__file__) -from zulint import lister from typing import Dict, Iterable, List +from zulint import lister + EXCLUDED_FILES = [ ## Test data Files for testing modules in tests "tools/tests/test_template_data", diff --git a/tools/check-thirdparty b/tools/check-thirdparty index 501b8f5c98f26..fc0f0b0175143 100755 --- a/tools/check-thirdparty +++ b/tools/check-thirdparty @@ -7,12 +7,12 @@ Disclaimer: This script is not a lawyer. It cannot validate that the claimed licenses are correct. It can only check for basic syntactic issues. """ - import difflib import io import os import subprocess import sys + from debian import copyright COPYRIGHT_FILENAME = "docs/THIRDPARTY" diff --git a/tools/create-test-api-docs b/tools/create-test-api-docs index 2c313659d2e4b..7ba2beb42151d 100755 --- a/tools/create-test-api-docs +++ b/tools/create-test-api-docs @@ -2,14 +2,15 @@ # check for the venv from lib import sanity_check -sanity_check.check_venv(__file__) -from collections import defaultdict -from typing import Any, Dict, List, Set +sanity_check.check_venv(__file__) import html import os import pprint +from collections import defaultdict +from typing import Any, Dict, List, Set + import ujson Call = Dict[str, Any] diff --git a/tools/diagnose b/tools/diagnose index 657ab87520534..80f1ce75e75d2 100755 --- a/tools/diagnose +++ b/tools/diagnose @@ -1,17 +1,16 @@ #!/usr/bin/env python3 - import os import platform import shlex -import sys import subprocess - +import sys from typing import Callable, List TOOLS_DIR = os.path.dirname(__file__) ROOT_DIR = os.path.dirname(TOOLS_DIR) sys.path.insert(0, ROOT_DIR) from scripts.lib.zulip_tools import get_dev_uuid_var_path + UUID_VAR_PATH = get_dev_uuid_var_path() def run(check_func: Callable[[], bool]) -> None: @@ -88,7 +87,7 @@ def test_models() -> bool: os.environ['DJANGO_SETTINGS_MODULE'] = settings_module import django django.setup() - from zerver.models import UserProfile, Realm + from zerver.models import Realm, UserProfile print('Num realms: ', Realm.objects.count()) print('Num users: ', UserProfile.objects.count()) return True diff --git a/tools/documentation_crawler/documentation_crawler/commands/crawl_with_status.py b/tools/documentation_crawler/documentation_crawler/commands/crawl_with_status.py index 7f51832fc1f41..010af602b10e5 100644 --- a/tools/documentation_crawler/documentation_crawler/commands/crawl_with_status.py +++ b/tools/documentation_crawler/documentation_crawler/commands/crawl_with_status.py @@ -1,8 +1,9 @@ import optparse -from scrapy.crawler import Crawler -from scrapy.commands import crawl from typing import List, Union +from scrapy.commands import crawl +from scrapy.crawler import Crawler + class Command(crawl.Command): def run(self, args: List[str], opts: optparse.Values) -> None: diff --git a/tools/documentation_crawler/documentation_crawler/spiders/check_documentation.py b/tools/documentation_crawler/documentation_crawler/spiders/check_documentation.py index 861553dad4913..86dce36313908 100755 --- a/tools/documentation_crawler/documentation_crawler/spiders/check_documentation.py +++ b/tools/documentation_crawler/documentation_crawler/spiders/check_documentation.py @@ -1,6 +1,5 @@ import os import pathlib - from typing import List from .common.spiders import BaseDocumentationSpider diff --git a/tools/documentation_crawler/documentation_crawler/spiders/check_help_documentation.py b/tools/documentation_crawler/documentation_crawler/spiders/check_help_documentation.py index 4b6bb43ac865c..082001eaee86f 100644 --- a/tools/documentation_crawler/documentation_crawler/spiders/check_help_documentation.py +++ b/tools/documentation_crawler/documentation_crawler/spiders/check_help_documentation.py @@ -1,12 +1,10 @@ import os - from posixpath import basename +from typing import Any, List, Set from urllib.parse import urlparse from .common.spiders import BaseDocumentationSpider -from typing import Any, List, Set - def get_images_dir(images_path: str) -> str: # Get index html file as start url and convert it to file uri diff --git a/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py b/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py index b75cdf8570c37..6d17c789a5d57 100644 --- a/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py +++ b/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py @@ -1,7 +1,8 @@ import json import re -import scrapy +from typing import Callable, Iterable, List, Optional, Union +import scrapy from scrapy.http import Request, Response from scrapy.linkextractors import IGNORED_EXTENSIONS from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor @@ -9,8 +10,6 @@ from scrapy.utils.url import url_has_any_extension from twisted.python.failure import Failure -from typing import Callable, Iterable, List, Optional, Union - EXCLUDED_URLS = [ # Google calendar returns 404s on HEAD requests unconditionally 'https://calendar.google.com/calendar/embed?src=ktiduof4eoh47lmgcl2qunnc0o@group.calendar.google.com', diff --git a/tools/droplets/add_mentor.py b/tools/droplets/add_mentor.py index f5058281202cd..75a977f973316 100644 --- a/tools/droplets/add_mentor.py +++ b/tools/droplets/add_mentor.py @@ -9,13 +9,12 @@ # machine: # # $ python3 add_mentor.py --remove - import os +import re +import socket import sys from argparse import ArgumentParser from typing import List -import socket -import re import requests diff --git a/tools/droplets/create.py b/tools/droplets/create.py index 0c2b8e6e2a718..320f01fbb40f8 100644 --- a/tools/droplets/create.py +++ b/tools/droplets/create.py @@ -14,19 +14,18 @@ # Copy conf.ini-template to conf.ini and populate with your api token. # # usage: python3 create.py - -import sys +import argparse import configparser -import urllib.error -import urllib.request import json -import digitalocean -import time -import argparse import os - +import sys +import time +import urllib.error +import urllib.request from typing import Any, Dict, List +import digitalocean + # initiation argument parser parser = argparse.ArgumentParser(description='Create a Zulip devopment VM Digital Ocean droplet.') parser.add_argument("username", help="Github username for whom you want to create a Zulip dev droplet") diff --git a/tools/fetch-contributor-data b/tools/fetch-contributor-data index 40a82d06fd255..327b921e44856 100755 --- a/tools/fetch-contributor-data +++ b/tools/fetch-contributor-data @@ -3,31 +3,32 @@ Fetch contributors data from Github using their API, convert it to structured JSON data for the /team page contributors section. """ - import os import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) from scripts.lib.setup_path import setup_path -setup_path() -from typing import Any, Dict, List, Optional, Union -from typing_extensions import TypedDict +setup_path() import argparse -from time import sleep +import logging from datetime import date from random import randrange -import logging +from time import sleep +from typing import Any, Dict, List, Optional, Union + +from typing_extensions import TypedDict os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.settings' import django + django.setup() -from django.conf import settings +import json import requests -import json +from django.conf import settings duplicate_commits_file = os.path.join(os.path.dirname(__file__), 'duplicate_commits.json') diff --git a/tools/generate-integration-docs-screenshot b/tools/generate-integration-docs-screenshot index c7b419e5119c1..44aefa6fd8c1e 100755 --- a/tools/generate-integration-docs-screenshot +++ b/tools/generate-integration-docs-screenshot @@ -5,7 +5,6 @@ from lib import sanity_check sanity_check.check_venv(__file__) - import os import sys @@ -26,23 +25,33 @@ import argparse import base64 import subprocess from typing import Any, Dict, Optional -from urllib.parse import urlencode, parse_qsl +from urllib.parse import parse_qsl, urlencode import requests import ujson from scripts.lib.zulip_tools import BOLDRED, ENDC from tools.lib.test_script import prepare_puppeteer_run -from zerver.models import UserProfile, Message, get_user_by_delivery_email, get_realm from zerver.lib.actions import ( - do_create_user, notify_created_bot, bulk_add_subscriptions, do_change_avatar_fields) + bulk_add_subscriptions, + do_change_avatar_fields, + do_create_user, + notify_created_bot, +) +from zerver.lib.integrations import ( + DOC_SCREENSHOT_CONFIG, + INTEGRATIONS, + ScreenshotConfig, + WebhookIntegration, + get_fixture_and_image_paths, + split_fixture_path, +) +from zerver.lib.storage import static_path from zerver.lib.streams import create_stream_if_needed from zerver.lib.upload import upload_avatar_image -from zerver.lib.integrations import ( - WebhookIntegration, INTEGRATIONS, split_fixture_path, ScreenshotConfig, get_fixture_and_image_paths, - DOC_SCREENSHOT_CONFIG) from zerver.lib.webhooks.common import get_fixture_http_headers -from zerver.lib.storage import static_path +from zerver.models import Message, UserProfile, get_realm, get_user_by_delivery_email + def create_integration_bot(integration: WebhookIntegration, bot_name: Optional[str]=None) -> UserProfile: realm = get_realm('zulip') diff --git a/tools/i18n/process-mobile-i18n b/tools/i18n/process-mobile-i18n index e8be13254d451..1ce108879eb42 100755 --- a/tools/i18n/process-mobile-i18n +++ b/tools/i18n/process-mobile-i18n @@ -5,6 +5,7 @@ import re from subprocess import check_output from typing import Dict, List + def get_json_filename(locale: str) -> str: return f"locale/{locale}/mobile.json" diff --git a/tools/i18n/tagmessages b/tools/i18n/tagmessages index 853a4d2044e05..9ad637d8b1ce0 100755 --- a/tools/i18n/tagmessages +++ b/tools/i18n/tagmessages @@ -1,8 +1,7 @@ #!/usr/bin/env python3 - import configparser -from hashlib import md5 import os +from hashlib import md5 import polib diff --git a/tools/lib/capitalization.py b/tools/lib/capitalization.py index 49aaba7db8cfc..3386d47cfc0f4 100644 --- a/tools/lib/capitalization.py +++ b/tools/lib/capitalization.py @@ -1,5 +1,5 @@ -from typing import List, Tuple, Match import re +from typing import List, Match, Tuple from bs4 import BeautifulSoup diff --git a/tools/lib/gitlint-rules.py b/tools/lib/gitlint-rules.py index 810267dd44ef2..de77c52cc5503 100644 --- a/tools/lib/gitlint-rules.py +++ b/tools/lib/gitlint-rules.py @@ -1,9 +1,9 @@ -from typing import Text, List +import re +from typing import List, Text from gitlint.git import GitCommit -from gitlint.rules import LineRule, RuleViolation, CommitMessageTitle from gitlint.options import StrOption -import re +from gitlint.rules import CommitMessageTitle, LineRule, RuleViolation # Word list from https://github.com/m1foley/fit-commit # Copyright (c) 2015 Mike Foley diff --git a/tools/lib/html_branches.py b/tools/lib/html_branches.py index fbfce1a2ecd14..ceb588e46df14 100644 --- a/tools/lib/html_branches.py +++ b/tools/lib/html_branches.py @@ -1,13 +1,8 @@ -from typing import Dict, List, Optional, Set - import re from collections import defaultdict +from typing import Dict, List, Optional, Set -from .template_parser import ( - tokenize, - FormattedException, - Token, -) +from .template_parser import FormattedException, Token, tokenize class HtmlBranchesException(Exception): diff --git a/tools/lib/pretty_print.py b/tools/lib/pretty_print.py index 6f72984f4bea7..2abc94ac2b08f 100644 --- a/tools/lib/pretty_print.py +++ b/tools/lib/pretty_print.py @@ -1,13 +1,10 @@ +import subprocess from typing import Any, Dict, List -from .template_parser import ( - tokenize, - is_django_block_tag, -) +from zulint.printer import ENDC, GREEN -from zulint.printer import GREEN, ENDC +from .template_parser import is_django_block_tag, tokenize -import subprocess def pretty_print_html(html: str, num_spaces: int = 4) -> str: # We use 1-based indexing for both rows and columns. diff --git a/tools/lib/provision.py b/tools/lib/provision.py index 7d949ba5ae3c1..b52b94d5241d7 100755 --- a/tools/lib/provision.py +++ b/tools/lib/provision.py @@ -1,28 +1,37 @@ #!/usr/bin/env python3 -import os -import sys -import logging import argparse +import hashlib +import logging +import os import platform import subprocess -import hashlib +import sys os.environ["PYTHONUNBUFFERED"] = "y" ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import run_as_root, ENDC, WARNING, \ - get_dev_uuid_var_path, FAIL, os_families, parse_os_release, \ - overwrite_symlink +from typing import TYPE_CHECKING, List + +from scripts.lib.node_cache import NODE_MODULES_CACHE_PATH, setup_node_modules from scripts.lib.setup_venv import ( - get_venv_dependencies, THUMBOR_VENV_DEPENDENCIES, + THUMBOR_VENV_DEPENDENCIES, YUM_THUMBOR_VENV_DEPENDENCIES, + get_venv_dependencies, +) +from scripts.lib.zulip_tools import ( + ENDC, + FAIL, + WARNING, + get_dev_uuid_var_path, + os_families, + overwrite_symlink, + parse_os_release, + run_as_root, ) -from scripts.lib.node_cache import setup_node_modules, NODE_MODULES_CACHE_PATH from tools.setup import setup_venvs -from typing import List, TYPE_CHECKING if TYPE_CHECKING: from typing import NoReturn diff --git a/tools/lib/provision_inner.py b/tools/lib/provision_inner.py index 3f09e9ed8e0a6..6582e2d2d82ac 100755 --- a/tools/lib/provision_inner.py +++ b/tools/lib/provision_inner.py @@ -1,22 +1,26 @@ #!/usr/bin/env python3 -import os -import sys import argparse import glob +import os import shutil - +import sys from typing import List ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import run, OKBLUE, ENDC, \ - get_dev_uuid_var_path, is_digest_obsolete, write_new_digest - -from version import PROVISION_VERSION from pygments import __version__ as pygments_version +from scripts.lib.zulip_tools import ( + ENDC, + OKBLUE, + get_dev_uuid_var_path, + is_digest_obsolete, + run, + write_new_digest, +) from tools.setup.generate_zulip_bots_static_files import generate_zulip_bots_static_files +from version import PROVISION_VERSION VENV_PATH = "/srv/zulip-py3-venv" UUID_VAR_PATH = get_dev_uuid_var_path() @@ -177,7 +181,7 @@ def clean_unused_caches() -> None: verbose=False, no_headings=True, ) - from scripts.lib import clean_venv_cache, clean_node_cache, clean_emoji_cache + from scripts.lib import clean_emoji_cache, clean_node_cache, clean_venv_cache clean_venv_cache.main(args) clean_node_cache.main(args) clean_emoji_cache.main(args) @@ -227,12 +231,13 @@ def main(options: argparse.Namespace) -> int: import django django.setup() + from django.conf import settings + from zerver.lib.test_fixtures import ( DEV_DATABASE, TEST_DATABASE, destroy_leaked_test_databases, ) - from django.conf import settings if options.is_force or need_to_run_configure_rabbitmq( [settings.RABBITMQ_PASSWORD]): diff --git a/tools/lib/sanity_check.py b/tools/lib/sanity_check.py index dac8209fdbe38..6ebe777b90ab5 100644 --- a/tools/lib/sanity_check.py +++ b/tools/lib/sanity_check.py @@ -2,6 +2,7 @@ import pwd import sys + def check_venv(filename: str) -> None: try: import django diff --git a/tools/lib/template_parser.py b/tools/lib/template_parser.py index 124f500dfde13..b53bde07ba694 100644 --- a/tools/lib/template_parser.py +++ b/tools/lib/template_parser.py @@ -1,5 +1,6 @@ from typing import Callable, List, Optional, Text + class FormattedException(Exception): pass diff --git a/tools/lib/test_script.py b/tools/lib/test_script.py index ca03ffae09ea3..5c7780262bee3 100644 --- a/tools/lib/test_script.py +++ b/tools/lib/test_script.py @@ -1,12 +1,12 @@ -from typing import Optional, Tuple, Iterable, List - +import glob import os import subprocess import sys from distutils.version import LooseVersion -from version import PROVISION_VERSION +from typing import Iterable, List, Optional, Tuple + from scripts.lib.zulip_tools import get_dev_uuid_var_path -import glob +from version import PROVISION_VERSION ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/tools/lib/test_server.py b/tools/lib/test_server.py index 5a1a66429eb59..5e47a8f343fa4 100644 --- a/tools/lib/test_server.py +++ b/tools/lib/test_server.py @@ -2,13 +2,12 @@ import subprocess import sys import time - from contextlib import contextmanager - from typing import Iterator, Optional # Verify the Zulip venv is available. from tools.lib import sanity_check + sanity_check.check_venv(__file__) import django @@ -20,8 +19,9 @@ if TOOLS_DIR not in sys.path: sys.path.insert(0, os.path.dirname(TOOLS_DIR)) -from zerver.lib.test_fixtures import update_test_databases_if_required from scripts.lib.zulip_tools import get_or_create_dev_uuid_var_path +from zerver.lib.test_fixtures import update_test_databases_if_required + def set_up_django(external_host: str) -> None: os.environ['EXTERNAL_HOST'] = external_host diff --git a/tools/lint b/tools/lint index e222cb6c087f3..2c379c83e5d02 100755 --- a/tools/lint +++ b/tools/lint @@ -1,16 +1,20 @@ #!/usr/bin/env python3 +import argparse import os import sys -import argparse # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) -from linter_lib.custom_check import python_rules, non_py_rules -from zulint.command import add_default_linter_arguments, LinterConfig import random +from zulint.command import LinterConfig, add_default_linter_arguments + +from linter_lib.custom_check import non_py_rules, python_rules + + def run() -> None: parser = argparse.ArgumentParser() parser.add_argument('--force', default=False, @@ -26,13 +30,10 @@ def run() -> None: root_dir = os.path.dirname(tools_dir) sys.path.insert(0, root_dir) + from tools.lib.test_script import assert_provisioning_status_ok from tools.linter_lib.exclude import EXCLUDED_FILES, PUPPET_CHECK_RULES_TO_EXCLUDE - from tools.linter_lib.pyflakes import check_pyflakes from tools.linter_lib.pep8 import check_pep8 - - from tools.lib.test_script import ( - assert_provisioning_status_ok, - ) + from tools.linter_lib.pyflakes import check_pyflakes os.chdir(root_dir) diff --git a/tools/linter_lib/pep8.py b/tools/linter_lib/pep8.py index 58e330462e733..93ad58de039d3 100644 --- a/tools/linter_lib/pep8.py +++ b/tools/linter_lib/pep8.py @@ -1,6 +1,7 @@ +from typing import List + from zulint.linters import run_pycodestyle -from typing import List def check_pep8(files: List[str]) -> bool: ignored_rules = [ diff --git a/tools/linter_lib/pyflakes.py b/tools/linter_lib/pyflakes.py index 33ecdd504f944..ff630b3a38f4c 100644 --- a/tools/linter_lib/pyflakes.py +++ b/tools/linter_lib/pyflakes.py @@ -1,5 +1,4 @@ import argparse - from typing import List from zulint.linters import run_pyflakes diff --git a/tools/pretty-print-html b/tools/pretty-print-html index d3fb88a4b38ac..c275c0440f21f 100755 --- a/tools/pretty-print-html +++ b/tools/pretty-print-html @@ -1,7 +1,9 @@ #!/usr/bin/env python3 +import sys from typing import List + from lib.pretty_print import pretty_print_html -import sys + def clean_html(filenames: List[str]) -> None: for fn in filenames: diff --git a/tools/renumber-migrations b/tools/renumber-migrations index 10998a8da7e50..600604ccbcdc9 100755 --- a/tools/renumber-migrations +++ b/tools/renumber-migrations @@ -1,13 +1,12 @@ #!/usr/bin/env python3 - +import fileinput import glob import os -import sys -import fileinput import re - +import sys from typing import List + def validate_order(order: List[int], length: int) -> None: if len(order) != length: print("Please enter the sequence of all the conflicting files at once") diff --git a/tools/review b/tools/review index 24541b4f0b5da..cf2f5b720861a 100755 --- a/tools/review +++ b/tools/review @@ -1,10 +1,10 @@ #!/usr/bin/env python3 - import shlex import subprocess import sys from typing import List + def exit(message: str) -> None: print('PROBLEM!') print(message) diff --git a/tools/run-dev.py b/tools/run-dev.py index 10615e874f3e8..8ca71c13c5e30 100755 --- a/tools/run-dev.py +++ b/tools/run-dev.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import argparse import os import pwd @@ -7,27 +6,22 @@ import subprocess import sys import traceback - from urllib.parse import urlunparse # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) -from tornado import httpclient -from tornado import httputil -from tornado import gen -from tornado import web +from tornado import gen, httpclient, httputil, web from tornado.ioloop import IOLoop TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.dirname(TOOLS_DIR)) -from tools.lib.test_script import ( - assert_provisioning_status_ok, -) - from typing import Any, Callable, Generator, List, Optional +from tools.lib.test_script import assert_provisioning_status_ok + if 'posix' in os.name and os.geteuid() == 0: raise RuntimeError("run-dev.py should not be run as root.") @@ -103,7 +97,7 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -from scripts.lib.zulip_tools import CYAN, WARNING, FAIL, ENDC +from scripts.lib.zulip_tools import CYAN, ENDC, FAIL, WARNING proxy_port = base_port django_port = base_port + 1 diff --git a/tools/run-mypy b/tools/run-mypy index 939b667616389..e72148591b293 100755 --- a/tools/run-mypy +++ b/tools/run-mypy @@ -1,10 +1,8 @@ #!/usr/bin/env python3 - - -import os -import sys import argparse +import os import subprocess +import sys from typing import List from zulint import lister diff --git a/tools/setup/build_pygments_data b/tools/setup/build_pygments_data index 7c15888c353d9..8b55ea399c28e 100755 --- a/tools/setup/build_pygments_data +++ b/tools/setup/build_pygments_data @@ -1,9 +1,9 @@ #!/usr/bin/env python3 - -from pygments.lexers import get_all_lexers import json import os +from pygments.lexers import get_all_lexers + ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../') DATA_PATH = os.path.join(ZULIP_PATH, 'tools', 'setup', 'lang.json') OUT_PATH = os.path.join(ZULIP_PATH, 'static', 'generated', 'pygments_data.json') diff --git a/tools/setup/emoji/build_emoji b/tools/setup/emoji/build_emoji index 23923323bf542..1ff34775df79c 100755 --- a/tools/setup/emoji/build_emoji +++ b/tools/setup/emoji/build_emoji @@ -5,14 +5,20 @@ import os import shutil import sys -import ujson - from typing import Any, Dict, Iterator, List, Optional -from emoji_setup_utils import generate_emoji_catalog, generate_codepoint_to_name_map, \ - get_emoji_code, generate_name_to_codepoint_map, emoji_names_for_picker, \ - EMOTICON_CONVERSIONS, REMAPPED_EMOJIS +import ujson + from emoji_names import EMOJI_NAME_MAPS +from emoji_setup_utils import ( + EMOTICON_CONVERSIONS, + REMAPPED_EMOJIS, + emoji_names_for_picker, + generate_codepoint_to_name_map, + generate_emoji_catalog, + generate_name_to_codepoint_map, + get_emoji_code, +) ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../') sys.path.append(ZULIP_PATH) diff --git a/tools/setup/emoji/emoji_setup_utils.py b/tools/setup/emoji/emoji_setup_utils.py index 64bdfbdb9e59d..3daff11a0b584 100644 --- a/tools/setup/emoji/emoji_setup_utils.py +++ b/tools/setup/emoji/emoji_setup_utils.py @@ -1,8 +1,6 @@ # This file contains various helper functions used by `build_emoji` tool. # See docs/subsystems/emoji.md for details on how this system works. - from collections import defaultdict - from typing import Any, Dict, List # Emojisets that we currently support. diff --git a/tools/setup/emoji/export_emoji_names_to_csv b/tools/setup/emoji/export_emoji_names_to_csv index da1ab25932d7c..c1e44a7e96bdc 100755 --- a/tools/setup/emoji/export_emoji_names_to_csv +++ b/tools/setup/emoji/export_emoji_names_to_csv @@ -10,12 +10,12 @@ import argparse import csv import os import re +from typing import Any, Dict, List + import ujson from emoji_setup_utils import get_emoji_code -from typing import Any, Dict, List - TOOLS_DIR_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ZULIP_PATH = os.path.dirname(TOOLS_DIR_PATH) # `emoji.json` file is same in all four emoji-datasource packages. diff --git a/tools/setup/emoji/generate_emoji_names_table b/tools/setup/emoji/generate_emoji_names_table index 71ecd8341e546..90a5ba1e116da 100755 --- a/tools/setup/emoji/generate_emoji_names_table +++ b/tools/setup/emoji/generate_emoji_names_table @@ -6,12 +6,12 @@ # sources' decisions about what names to provide to each unicode # codepoint. import os -import ujson - from typing import Any, Dict, List -from emoji_setup_utils import emoji_is_universal, get_emoji_code, EMOJISETS +import ujson + from emoji_names import EMOJI_NAME_MAPS +from emoji_setup_utils import EMOJISETS, emoji_is_universal, get_emoji_code TOOLS_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ZULIP_PATH = os.path.dirname(TOOLS_DIR) diff --git a/tools/setup/emoji/import_emoji_names_from_csv b/tools/setup/emoji/import_emoji_names_from_csv index f59977f35918f..cd188c47b2648 100755 --- a/tools/setup/emoji/import_emoji_names_from_csv +++ b/tools/setup/emoji/import_emoji_names_from_csv @@ -13,7 +13,6 @@ import csv import os import re import textwrap - from typing import Any, Dict, List, Set EMOJI_DIR_PATH = os.path.dirname(os.path.abspath(__file__)) diff --git a/tools/setup/emoji/test-emoji-name-scripts b/tools/setup/emoji/test-emoji-name-scripts index 0dce85763f4e9..1cd7b9b1d6d94 100755 --- a/tools/setup/emoji/test-emoji-name-scripts +++ b/tools/setup/emoji/test-emoji-name-scripts @@ -3,8 +3,8 @@ import argparse import difflib import filecmp import os -import subprocess import shutil +import subprocess import tempfile TOOLS_DIR = os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) diff --git a/tools/setup/generate_integration_bots_avatars.py b/tools/setup/generate_integration_bots_avatars.py index b5b38a1e6d394..9cd2f6b8ebcac 100755 --- a/tools/setup/generate_integration_bots_avatars.py +++ b/tools/setup/generate_integration_bots_avatars.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import os import sys @@ -7,6 +6,7 @@ if ZULIP_PATH not in sys.path: sys.path.append(ZULIP_PATH) from scripts.lib.setup_path import setup_path + setup_path() os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings" @@ -20,9 +20,10 @@ import cairosvg from PIL import Image -from zerver.lib.upload import resize_avatar, DEFAULT_AVATAR_SIZE from zerver.lib.integrations import WEBHOOK_INTEGRATIONS from zerver.lib.storage import static_path +from zerver.lib.upload import DEFAULT_AVATAR_SIZE, resize_avatar + def create_square_image(png: bytes) -> bytes: img = Image.open(io.BytesIO(png)) diff --git a/tools/setup/generate_zulip_bots_static_files.py b/tools/setup/generate_zulip_bots_static_files.py index 83c22fbd7b731..17ba1f3d1db14 100755 --- a/tools/setup/generate_zulip_bots_static_files.py +++ b/tools/setup/generate_zulip_bots_static_files.py @@ -1,19 +1,20 @@ #!/usr/bin/env python3 - import glob import os -import sys import shutil +import sys from typing import List ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) if ZULIP_PATH not in sys.path: sys.path.append(ZULIP_PATH) from scripts.lib.setup_path import setup_path + setup_path() from zulip_bots.lib import get_bots_directory_path + def generate_zulip_bots_static_files() -> None: bots_dir = 'static/generated/bots' if os.path.isdir(bots_dir): diff --git a/tools/show-profile-results b/tools/show-profile-results index 0ac294cc4f2b0..fb6aa1ce0f9a2 100755 --- a/tools/show-profile-results +++ b/tools/show-profile-results @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import sys import pstats +import sys ''' This is a helper script to make it easy to show profile diff --git a/tools/test-api b/tools/test-api index 7685dd012e303..ef61dbbc56546 100755 --- a/tools/test-api +++ b/tools/test-api @@ -7,6 +7,7 @@ os.environ["RUNNING_OPENAPI_CURL_TEST"] = "1" # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -30,13 +31,13 @@ assert_provisioning_status_ok(options.force) with test_server_running(force=options.force, external_host='zulipdev.com:9981'): # Zerver imports should happen after `django.setup()` is run # by the test_server_running decorator. - from zerver.openapi.python_examples import test_the_api, test_invalid_api_key - from zerver.openapi.javascript_examples import test_js_bindings - from zerver.openapi.test_curl_examples import test_generated_curl_examples_for_success from zerver.lib.actions import do_create_user from zerver.lib.test_helpers import reset_emails_in_zulip_realm from zerver.lib.users import get_api_key - from zerver.models import get_user, get_realm + from zerver.models import get_realm, get_user + from zerver.openapi.javascript_examples import test_js_bindings + from zerver.openapi.python_examples import test_invalid_api_key, test_the_api + from zerver.openapi.test_curl_examples import test_generated_curl_examples_for_success print("Running API tests...") diff --git a/tools/test-backend b/tools/test-backend index 6f7830849dc15..8c8fc3b1576cc 100755 --- a/tools/test-backend +++ b/tools/test-backend @@ -2,22 +2,23 @@ # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) -from typing import List, Iterator -import glob import argparse import contextlib -from unittest import mock +import glob import os import shlex -import sys import subprocess +import sys import tempfile -import ujson -import responses +from typing import Iterator, List +from unittest import mock import django +import responses +import ujson from django.conf import settings from django.test.utils import get_runner @@ -183,11 +184,10 @@ def main() -> None: os.environ.pop("http_proxy", "") os.environ.pop("https_proxy", "") - from zerver.lib.test_fixtures import update_test_databases_if_required, \ - remove_test_run_directories - - from tools.lib.test_script import ( - assert_provisioning_status_ok, + from tools.lib.test_script import assert_provisioning_status_ok + from zerver.lib.test_fixtures import ( + remove_test_run_directories, + update_test_databases_if_required, ) os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.test_settings' @@ -460,6 +460,7 @@ def main() -> None: if options.report_slow_tests: from zerver.lib.test_runner import report_slow_tests + # We do this even with failures, since slowness can be # an important clue as to why tests fail. report_slow_tests() diff --git a/tools/test-help-documentation b/tools/test-help-documentation index 7977c2b1c4a6b..e0bb8aa4dc083 100755 --- a/tools/test-help-documentation +++ b/tools/test-help-documentation @@ -2,14 +2,15 @@ import argparse import contextlib import os -import sys import subprocess +import sys from typing import Iterator ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) parser = argparse.ArgumentParser() diff --git a/tools/test-js-with-node b/tools/test-js-with-node index 0f9a16d55c07f..ca9b70c45675d 100755 --- a/tools/test-js-with-node +++ b/tools/test-js-with-node @@ -5,7 +5,7 @@ import os import pwd import subprocess import sys -from typing import Dict, Any +from typing import Any, Dict TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.dirname(TOOLS_DIR)) @@ -13,12 +13,12 @@ ROOT_DIR = os.path.dirname(TOOLS_DIR) # check for the venv from tools.lib import sanity_check + sanity_check.check_venv(__file__) # Import this after we do the sanity_check so it doesn't crash. import ujson - -from zulint.printer import CYAN, GREEN, BOLDRED, ENDC +from zulint.printer import BOLDRED, CYAN, ENDC, GREEN INDEX_JS = 'frontend_tests/zjsunit/index.js' NODE_COVERAGE_PATH = 'var/node-coverage/coverage-final.json' diff --git a/tools/test-js-with-puppeteer b/tools/test-js-with-puppeteer index fc2219b67c47c..9938079c46a5a 100755 --- a/tools/test-js-with-puppeteer +++ b/tools/test-js-with-puppeteer @@ -1,9 +1,9 @@ #!/usr/bin/env python3 import argparse -import subprocess -import sys import os import shlex +import subprocess +import sys ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -42,12 +42,18 @@ sys.path.insert(0, ZULIP_PATH) # check for the venv from tools.lib import sanity_check + sanity_check.check_venv(__file__) -from tools.lib.test_script import assert_provisioning_status_ok, find_js_test_files, prepare_puppeteer_run +from typing import Iterable + +from tools.lib.test_script import ( + assert_provisioning_status_ok, + find_js_test_files, + prepare_puppeteer_run, +) from tools.lib.test_server import test_server_running -from typing import Iterable def run_tests(files: Iterable[str], external_host: str) -> None: test_dir = os.path.join(ZULIP_PATH, 'frontend_tests/puppeteer_tests') diff --git a/tools/test-locked-requirements b/tools/test-locked-requirements index 98cc73ee4170f..2390264660712 100755 --- a/tools/test-locked-requirements +++ b/tools/test-locked-requirements @@ -8,9 +8,9 @@ import shutil import subprocess import sys import tempfile -import ujson +from typing import List, Optional -from typing import Optional, List +import ujson TOOLS_DIR = os.path.abspath(os.path.dirname(__file__)) ZULIP_PATH = os.path.dirname(TOOLS_DIR) diff --git a/tools/test-queue-worker-reload b/tools/test-queue-worker-reload index 152edf202e357..6f64dd7d8e55e 100755 --- a/tools/test-queue-worker-reload +++ b/tools/test-queue-worker-reload @@ -1,15 +1,14 @@ #!/usr/bin/env python3 - - import os -import sys -import time import signal import subprocess +import sys +import time import types # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) # TODO: Convert this to use scripts/lib/queue_workers.py diff --git a/tools/test-run-dev b/tools/test-run-dev index 9835c27021e91..f7d882373f702 100755 --- a/tools/test-run-dev +++ b/tools/test-run-dev @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import os import signal import subprocess @@ -8,6 +7,7 @@ import time from typing import Tuple from lib import sanity_check + sanity_check.check_venv(__file__) TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/tools/test-tools b/tools/test-tools index 14775fd74c4b1..326f726ecff88 100755 --- a/tools/test-tools +++ b/tools/test-tools @@ -1,6 +1,4 @@ #!/usr/bin/env python3 - - import argparse import os import sys @@ -8,6 +6,7 @@ import unittest # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) if __name__ == '__main__': diff --git a/tools/tests/test_capitalization_checker.py b/tools/tests/test_capitalization_checker.py index 37096d739101b..2789f653c7604 100644 --- a/tools/tests/test_capitalization_checker.py +++ b/tools/tests/test_capitalization_checker.py @@ -1,8 +1,9 @@ -from bs4 import BeautifulSoup from unittest import TestCase -from tools.lib.capitalization import check_capitalization, is_capitalized, \ - get_safe_text +from bs4 import BeautifulSoup + +from tools.lib.capitalization import check_capitalization, get_safe_text, is_capitalized + class GetSafeTextTestCase(TestCase): def test_get_safe_text(self) -> None: diff --git a/tools/tests/test_check_rabbitmq_queue.py b/tools/tests/test_check_rabbitmq_queue.py index d135097194626..2eb531e1d90aa 100644 --- a/tools/tests/test_check_rabbitmq_queue.py +++ b/tools/tests/test_check_rabbitmq_queue.py @@ -1,14 +1,8 @@ -from unittest import mock -from unittest import TestCase - -from scripts.lib.check_rabbitmq_queue import ( - analyze_queue_stats, - OK, - WARNING, - CRITICAL, - UNKNOWN, -) import time +from unittest import TestCase, mock + +from scripts.lib.check_rabbitmq_queue import CRITICAL, OK, UNKNOWN, WARNING, analyze_queue_stats + class AnalyzeQueueStatsTests(TestCase): def test_no_stats_available(self) -> None: diff --git a/tools/tests/test_html_branches.py b/tools/tests/test_html_branches.py index 7db3609124e83..36d268a255b91 100644 --- a/tools/tests/test_html_branches.py +++ b/tools/tests/test_html_branches.py @@ -1,13 +1,12 @@ -import unittest import os +import unittest import tools.lib.template_parser - from tools.lib.html_branches import ( + build_id_dict, get_tag_info, html_branches, html_tag_tree, - build_id_dict, split_for_id_and_class, ) diff --git a/tools/tests/test_template_parser.py b/tools/tests/test_template_parser.py index 5928101e47acd..421a10a1a3e2d 100644 --- a/tools/tests/test_template_parser.py +++ b/tools/tests/test_template_parser.py @@ -1,7 +1,6 @@ -from typing import Optional - import sys import unittest +from typing import Optional try: from tools.lib.template_parser import ( diff --git a/tools/tests/test_zulint_custom_rules.py b/tools/tests/test_zulint_custom_rules.py index ce11d9ea7d3df..f858e8de6005a 100644 --- a/tools/tests/test_zulint_custom_rules.py +++ b/tools/tests/test_zulint_custom_rules.py @@ -1,11 +1,11 @@ -from io import StringIO import os - -from unittest.mock import patch +from io import StringIO from unittest import TestCase +from unittest.mock import patch from zulint.custom_rules import RuleList -from linter_lib.custom_check import python_rules, non_py_rules + +from linter_lib.custom_check import non_py_rules, python_rules ROOT_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..')) CHECK_MESSAGE = "Fix the corresponding rule in `tools/linter_lib/custom_check.py`." diff --git a/tools/update-prod-static b/tools/update-prod-static index 1acc64781ca9c..c42f45a3a0e68 100755 --- a/tools/update-prod-static +++ b/tools/update-prod-static @@ -1,10 +1,8 @@ #!/usr/bin/env python3 # Updates static files for production. - - -import os import argparse +import os import sys # We need settings so we can figure out where the prod-static directory is. @@ -15,11 +13,12 @@ setup_path() os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.settings' from django.conf import settings -from scripts.lib.node_cache import setup_node_modules -from scripts.lib.zulip_tools import run # check for the venv from lib import sanity_check +from scripts.lib.node_cache import setup_node_modules +from scripts.lib.zulip_tools import run + sanity_check.check_venv(__file__) parser = argparse.ArgumentParser() diff --git a/tools/webpack b/tools/webpack index 1187e0bb42905..7ff053b14f85c 100755 --- a/tools/webpack +++ b/tools/webpack @@ -1,8 +1,7 @@ #!/usr/bin/env python3 - import argparse -import os import json +import os import subprocess from typing import NoReturn diff --git a/tools/zulip-export/zulip-export b/tools/zulip-export/zulip-export index 3072588c3911a..b65dec3eedb68 100755 --- a/tools/zulip-export/zulip-export +++ b/tools/zulip-export/zulip-export @@ -19,11 +19,11 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. - -import sys -import os import argparse import json +import os +import sys + usage = """Export all messages on a given stream to a JSON dump. zulip-export --user= --api-key= --stream= diff --git a/zerver/apps.py b/zerver/apps.py index d1b95a3827cde..4d6d355daff05 100644 --- a/zerver/apps.py +++ b/zerver/apps.py @@ -6,6 +6,7 @@ from django.core.cache import cache from django.db.models.signals import post_migrate + def flush_cache(sender: AppConfig, **kwargs: Any) -> None: logging.info("Clearing memcached cache after migrations") cache.clear() diff --git a/zerver/context_processors.py b/zerver/context_processors.py index a0bda24c60d7f..34bbc538dc7bb 100644 --- a/zerver/context_processors.py +++ b/zerver/context_processors.py @@ -1,27 +1,31 @@ +from typing import Any, Dict, Optional from urllib.parse import urljoin -from typing import Any, Dict, Optional -from django.http import HttpRequest from django.conf import settings +from django.http import HttpRequest -from zerver.models import UserProfile, get_realm, Realm +from version import ( + LATEST_MAJOR_VERSION, + LATEST_RELEASE_ANNOUNCEMENT, + LATEST_RELEASE_VERSION, + ZULIP_VERSION, +) +from zerver.decorator import get_client_name +from zerver.lib.realm_description import get_realm_rendered_description, get_realm_text_description +from zerver.lib.realm_icon import get_realm_icon_url +from zerver.lib.send_email import FromAddress +from zerver.lib.subdomains import get_subdomain +from zerver.models import Realm, UserProfile, get_realm from zproject.backends import ( + AUTH_BACKEND_NAME_MAP, + AppleAuthBackend, any_social_backend_enabled, + auth_enabled_helper, get_external_method_dicts, password_auth_enabled, require_email_format_usernames, - auth_enabled_helper, - AUTH_BACKEND_NAME_MAP, - AppleAuthBackend, ) -from zerver.decorator import get_client_name -from zerver.lib.send_email import FromAddress -from zerver.lib.subdomains import get_subdomain -from zerver.lib.realm_icon import get_realm_icon_url -from zerver.lib.realm_description import get_realm_rendered_description, get_realm_text_description -from version import ZULIP_VERSION, LATEST_RELEASE_VERSION, LATEST_MAJOR_VERSION, \ - LATEST_RELEASE_ANNOUNCEMENT def common_context(user: UserProfile) -> Dict[str, Any]: """Common context used for things like outgoing emails that don't diff --git a/zerver/data_import/gitter.py b/zerver/data_import/gitter.py index 1dd10fa8c2d7e..b168f528541bc 100644 --- a/zerver/data_import/gitter.py +++ b/zerver/data_import/gitter.py @@ -1,20 +1,31 @@ -import os -import dateutil.parser import logging +import os import subprocess -import ujson +from typing import Any, Dict, List, Set, Tuple +import dateutil.parser +import ujson from django.conf import settings from django.forms.models import model_to_dict from django.utils.timezone import now as timezone_now -from typing import Any, Dict, List, Set, Tuple -from zerver.models import UserProfile, Recipient +from zerver.data_import.import_util import ( + ZerverFieldsT, + build_avatar, + build_defaultstream, + build_message, + build_realm, + build_recipient, + build_stream, + build_subscription, + build_usermessages, + build_zerver_realm, + create_converted_data_files, + make_subscriber_map, + process_avatars, +) from zerver.lib.export import MESSAGE_BATCH_CHUNK_SIZE -from zerver.data_import.import_util import ZerverFieldsT, build_zerver_realm, \ - build_avatar, build_subscription, build_recipient, build_usermessages, \ - build_defaultstream, process_avatars, build_realm, build_stream, \ - build_message, create_converted_data_files, make_subscriber_map +from zerver.models import Recipient, UserProfile # stubs GitterDataT = List[Dict[str, Any]] diff --git a/zerver/data_import/hipchat.py b/zerver/data_import/hipchat.py index 92cadb4f3b087..386672bf1085e 100755 --- a/zerver/data_import/hipchat.py +++ b/zerver/data_import/hipchat.py @@ -1,37 +1,29 @@ import base64 -import dateutil import glob -import hypchat import logging import os import re import shutil import subprocess -import ujson - from typing import Any, Callable, Dict, List, Optional, Set +import dateutil +import hypchat +import ujson from django.conf import settings from django.utils.timezone import now as timezone_now -from zerver.lib.utils import ( - process_list_in_batches, -) - -from zerver.models import ( - RealmEmoji, - Recipient, - UserProfile, -) - +from zerver.data_import.hipchat_attachment import AttachmentHandler +from zerver.data_import.hipchat_user import UserHandler from zerver.data_import.import_util import ( + SubscriberHandler, build_message, + build_personal_subscriptions, + build_public_stream_subscriptions, build_realm, build_realm_emoji, build_recipients, build_stream, - build_personal_subscriptions, - build_public_stream_subscriptions, build_stream_subscriptions, build_user_profile, build_zerver_realm, @@ -39,12 +31,10 @@ make_subscriber_map, make_user_messages, write_avatar_png, - SubscriberHandler, ) - -from zerver.data_import.hipchat_attachment import AttachmentHandler -from zerver.data_import.hipchat_user import UserHandler from zerver.data_import.sequencer import NEXT_ID, IdMapper +from zerver.lib.utils import process_list_in_batches +from zerver.models import RealmEmoji, Recipient, UserProfile # stubs ZerverFieldsT = Dict[str, Any] diff --git a/zerver/data_import/hipchat_attachment.py b/zerver/data_import/hipchat_attachment.py index de676751dd415..596d3e588f466 100644 --- a/zerver/data_import/hipchat_attachment.py +++ b/zerver/data_import/hipchat_attachment.py @@ -1,13 +1,10 @@ import logging -import shutil import os +import shutil +from typing import Any, Dict, List, Optional -from zerver.data_import.import_util import ( - build_attachment, - create_converted_data_files, -) +from zerver.data_import.import_util import build_attachment, create_converted_data_files -from typing import Any, Dict, List, Optional class AttachmentHandler: def __init__(self) -> None: diff --git a/zerver/data_import/hipchat_user.py b/zerver/data_import/hipchat_user.py index 10b3950dee5d3..ef03ece81bada 100644 --- a/zerver/data_import/hipchat_user.py +++ b/zerver/data_import/hipchat_user.py @@ -2,12 +2,9 @@ from django.utils.timezone import now as timezone_now -from zerver.data_import.import_util import ( - build_user_profile, -) -from zerver.models import ( - UserProfile, -) +from zerver.data_import.import_util import build_user_profile +from zerver.models import UserProfile + class UserHandler: ''' diff --git a/zerver/data_import/import_util.py b/zerver/data_import/import_util.py index f7580e454ef1a..31f235df09db8 100644 --- a/zerver/data_import/import_util.py +++ b/zerver/data_import/import_util.py @@ -1,20 +1,29 @@ -import random -import requests -import shutil import logging import os +import random +import shutil import traceback -import ujson +from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, TypeVar -from typing import List, Dict, Any, Optional, Set, Callable, Iterable, Tuple, TypeVar +import requests +import ujson from django.forms.models import model_to_dict -from zerver.models import Realm, RealmEmoji, Subscription, Recipient, \ - Attachment, Stream, Message, UserProfile, Huddle from zerver.data_import.sequencer import NEXT_ID from zerver.lib.actions import STREAM_ASSIGNMENT_COLORS as stream_colors from zerver.lib.avatar_hash import user_avatar_path_from_ids from zerver.lib.parallel import run_parallel +from zerver.models import ( + Attachment, + Huddle, + Message, + Realm, + RealmEmoji, + Recipient, + Stream, + Subscription, + UserProfile, +) # stubs ZerverFieldsT = Dict[str, Any] diff --git a/zerver/data_import/mattermost.py b/zerver/data_import/mattermost.py index 1bb4dfa3beff5..ba1823a4fc40c 100644 --- a/zerver/data_import/mattermost.py +++ b/zerver/data_import/mattermost.py @@ -2,33 +2,42 @@ spec: https://docs.mattermost.com/administration/bulk-export.html """ -import os import logging -import subprocess -import ujson +import os import re import shutil - +import subprocess from typing import Any, Callable, Dict, List, Set +import ujson from django.conf import settings -from django.utils.timezone import now as timezone_now from django.forms.models import model_to_dict +from django.utils.timezone import now as timezone_now -from zerver.models import Recipient, RealmEmoji, Reaction, UserProfile -from zerver.lib.utils import ( - process_list_in_batches, +from zerver.data_import.import_util import ( + SubscriberHandler, + ZerverFieldsT, + build_huddle, + build_huddle_subscriptions, + build_message, + build_personal_subscriptions, + build_realm, + build_realm_emoji, + build_recipients, + build_stream, + build_stream_subscriptions, + build_user_profile, + build_zerver_realm, + create_converted_data_files, + make_subscriber_map, + make_user_messages, ) -from zerver.lib.emoji import name_to_codepoint -from zerver.data_import.import_util import ZerverFieldsT, build_zerver_realm, \ - build_stream, build_realm, build_message, create_converted_data_files, \ - make_subscriber_map, build_recipients, build_user_profile, \ - build_stream_subscriptions, build_huddle_subscriptions, \ - build_personal_subscriptions, SubscriberHandler, \ - build_realm_emoji, make_user_messages, build_huddle - from zerver.data_import.mattermost_user import UserHandler from zerver.data_import.sequencer import NEXT_ID, IdMapper +from zerver.lib.emoji import name_to_codepoint +from zerver.lib.utils import process_list_in_batches +from zerver.models import Reaction, RealmEmoji, Recipient, UserProfile + def make_realm(realm_id: int, team: Dict[str, Any]) -> ZerverFieldsT: # set correct realm details diff --git a/zerver/data_import/mattermost_user.py b/zerver/data_import/mattermost_user.py index c8585584a1b5c..e6d629314ee62 100644 --- a/zerver/data_import/mattermost_user.py +++ b/zerver/data_import/mattermost_user.py @@ -1,5 +1,6 @@ from typing import Any, Dict, List + class UserHandler: ''' Our UserHandler class is a glorified wrapper diff --git a/zerver/data_import/slack.py b/zerver/data_import/slack.py index 71b487dde7109..798345cd8b754 100755 --- a/zerver/data_import/slack.py +++ b/zerver/data_import/slack.py @@ -1,32 +1,54 @@ +import logging import os -import ujson +import random import shutil import subprocess -import logging -import random -import requests - from collections import defaultdict +from typing import Any, Dict, Iterator, List, Optional, Set, Tuple +from urllib.parse import urlencode +import requests +import ujson from django.conf import settings -from django.utils.timezone import now as timezone_now from django.forms.models import model_to_dict -from typing import Any, Dict, List, Optional, Tuple, Set, Iterator -from zerver.models import Reaction, RealmEmoji, UserProfile, Recipient, \ - CustomProfileField, CustomProfileFieldValue, Realm -from zerver.data_import.slack_message_conversion import convert_to_zulip_markdown, \ - get_user_full_name -from zerver.data_import.import_util import ZerverFieldsT, build_zerver_realm, \ - build_avatar, build_subscription, build_recipient, build_usermessages, \ - build_defaultstream, build_attachment, process_avatars, process_uploads, \ - process_emojis, build_realm, build_stream, build_huddle, build_message, \ - create_converted_data_files, make_subscriber_map +from django.utils.timezone import now as timezone_now + +from zerver.data_import.import_util import ( + ZerverFieldsT, + build_attachment, + build_avatar, + build_defaultstream, + build_huddle, + build_message, + build_realm, + build_recipient, + build_stream, + build_subscription, + build_usermessages, + build_zerver_realm, + create_converted_data_files, + make_subscriber_map, + process_avatars, + process_emojis, + process_uploads, +) from zerver.data_import.sequencer import NEXT_ID -from zerver.lib.upload import random_name, sanitize_name -from zerver.lib.export import MESSAGE_BATCH_CHUNK_SIZE +from zerver.data_import.slack_message_conversion import ( + convert_to_zulip_markdown, + get_user_full_name, +) from zerver.lib.emoji import name_to_codepoint -from zerver.lib.upload import resize_logo -from urllib.parse import urlencode +from zerver.lib.export import MESSAGE_BATCH_CHUNK_SIZE +from zerver.lib.upload import random_name, resize_logo, sanitize_name +from zerver.models import ( + CustomProfileField, + CustomProfileFieldValue, + Reaction, + Realm, + RealmEmoji, + Recipient, + UserProfile, +) SlackToZulipUserIDT = Dict[str, int] AddedChannelsT = Dict[str, Tuple[str, int]] diff --git a/zerver/data_import/slack_message_conversion.py b/zerver/data_import/slack_message_conversion.py index 0905541568cd3..bfd8a54eb2c2b 100644 --- a/zerver/data_import/slack_message_conversion.py +++ b/zerver/data_import/slack_message_conversion.py @@ -1,5 +1,5 @@ import re -from typing import Any, Dict, Tuple, List, Optional +from typing import Any, Dict, List, Optional, Tuple # stubs ZerverFieldsT = Dict[str, Any] diff --git a/zerver/decorator.py b/zerver/decorator.py index f87eacdccc252..ae4a62d231b93 100644 --- a/zerver/decorator.py +++ b/zerver/decorator.py @@ -1,51 +1,60 @@ -import django_otp -from two_factor.utils import default_device -from django_otp import user_has_device +import base64 +import datetime +import logging +import urllib +from functools import wraps +from io import BytesIO +from typing import Any, Callable, Dict, Optional, Tuple, TypeVar, Union +import django_otp +import ujson +from django.conf import settings +from django.contrib.auth import REDIRECT_FIELD_NAME +from django.contrib.auth import login as django_login from django.contrib.auth.decorators import user_passes_test as django_user_passes_test from django.contrib.auth.models import AnonymousUser -from django.utils.translation import ugettext as _ -from django.http import HttpResponseRedirect, HttpResponse -from django.contrib.auth import REDIRECT_FIELD_NAME, login as django_login -from django.views.decorators.csrf import csrf_exempt -from django.http import QueryDict, HttpResponseNotAllowed, HttpRequest +from django.http import ( + HttpRequest, + HttpResponse, + HttpResponseNotAllowed, + HttpResponseRedirect, + QueryDict, +) from django.http.multipartparser import MultiPartParser -from zerver.models import Realm, UserProfile, get_client, get_user_profile_by_api_key -from zerver.lib.response import json_error, json_unauthorized, json_success from django.shortcuts import resolve_url +from django.template.response import SimpleTemplateResponse from django.utils.decorators import available_attrs from django.utils.timezone import now as timezone_now -from django.conf import settings -from django.template.response import SimpleTemplateResponse +from django.utils.translation import ugettext as _ +from django.views.decorators.csrf import csrf_exempt +from django_otp import user_has_device +from two_factor.utils import default_device -from zerver.lib.exceptions import UnexpectedWebhookEventType +from zerver.lib.exceptions import ( + ErrorCode, + InvalidAPIKeyError, + InvalidAPIKeyFormatError, + InvalidJSONError, + JsonableError, + OrganizationAdministratorRequired, + OrganizationOwnerRequired, + UnexpectedWebhookEventType, +) +from zerver.lib.logging_util import log_to_file from zerver.lib.queue import queue_json_publish +from zerver.lib.rate_limiter import RateLimitedUser +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_error, json_success, json_unauthorized from zerver.lib.subdomains import get_subdomain, user_matches_subdomain from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime -from zerver.lib.utils import statsd, has_api_key_format -from zerver.lib.exceptions import JsonableError, ErrorCode, \ - InvalidJSONError, InvalidAPIKeyError, InvalidAPIKeyFormatError, \ - OrganizationAdministratorRequired, OrganizationOwnerRequired from zerver.lib.types import ViewFuncT - -from zerver.lib.rate_limiter import RateLimitedUser -from zerver.lib.request import REQ, has_request_variables - -from functools import wraps -import base64 -import datetime -import ujson -import logging -from io import BytesIO -import urllib - -from typing import Union, Any, Callable, Dict, Optional, TypeVar, Tuple -from zerver.lib.logging_util import log_to_file +from zerver.lib.utils import has_api_key_format, statsd +from zerver.models import Realm, UserProfile, get_client, get_user_profile_by_api_key # This is a hack to ensure that RemoteZulipServer always exists even # if Zilencer isn't enabled. if settings.ZILENCER_ENABLED: - from zilencer.models import get_remote_server_by_uuid, RemoteZulipServer + from zilencer.models import RemoteZulipServer, get_remote_server_by_uuid else: # nocoverage # Hack here basically to make impossible code paths compile from unittest.mock import Mock get_remote_server_by_uuid = Mock() @@ -130,6 +139,7 @@ def wrapper(request: HttpRequest, user_profile: UserProfile, *args: Any, **kwarg from zerver.lib.user_agent import parse_user_agent + def get_client_name(request: HttpRequest) -> str: # If the API request specified a client in the request content, # that has priority. Otherwise, extract the client from the diff --git a/zerver/filters.py b/zerver/filters.py index f74eae3aae5b3..8d3e2fbe47dc0 100644 --- a/zerver/filters.py +++ b/zerver/filters.py @@ -4,6 +4,7 @@ from django.http import HttpRequest from django.views.debug import SafeExceptionReporterFilter + class ZulipExceptionReporterFilter(SafeExceptionReporterFilter): def get_post_parameters(self, request: HttpRequest) -> Dict[str, Any]: filtered_post = SafeExceptionReporterFilter.get_post_parameters(self, request).copy() diff --git a/zerver/forms.py b/zerver/forms.py index 6b70e7f67c769..2f1cc9ca1deec 100644 --- a/zerver/forms.py +++ b/zerver/forms.py @@ -1,41 +1,43 @@ +import logging +import re +from typing import Any, Dict, List, Optional, Tuple + +import DNS from django import forms from django.conf import settings from django.contrib.auth import authenticate, password_validation -from django.contrib.auth.forms import SetPasswordForm, AuthenticationForm, \ - PasswordResetForm +from django.contrib.auth.forms import AuthenticationForm, PasswordResetForm, SetPasswordForm +from django.contrib.auth.tokens import PasswordResetTokenGenerator, default_token_generator from django.core.exceptions import ValidationError -from django.urls import reverse from django.core.validators import validate_email -from django.utils.translation import ugettext as _ -from django.contrib.auth.tokens import default_token_generator -from django.utils.http import urlsafe_base64_encode -from django.utils.encoding import force_bytes -from django.contrib.auth.tokens import PasswordResetTokenGenerator from django.http import HttpRequest +from django.urls import reverse +from django.utils.encoding import force_bytes +from django.utils.http import urlsafe_base64_encode +from django.utils.translation import ugettext as _ from jinja2 import Markup as mark_safe +from two_factor.forms import AuthenticationTokenForm as TwoFactorAuthenticationTokenForm +from two_factor.utils import totp_digits from zerver.lib.actions import do_change_password, email_not_system_bot -from zerver.lib.email_validation import email_allowed_for_realm, \ - validate_email_not_already_in_realm -from zerver.lib.name_restrictions import is_reserved_subdomain, is_disposable_domain +from zerver.lib.email_validation import email_allowed_for_realm, validate_email_not_already_in_realm +from zerver.lib.name_restrictions import is_disposable_domain, is_reserved_subdomain from zerver.lib.rate_limiter import RateLimited, RateLimitedObject from zerver.lib.request import JsonableError -from zerver.lib.send_email import send_email, FromAddress +from zerver.lib.send_email import FromAddress, send_email from zerver.lib.subdomains import get_subdomain, is_root_domain_available from zerver.lib.users import check_full_name -from zerver.models import Realm, get_user_by_delivery_email, UserProfile, get_realm, \ - email_to_domain, \ - DisposableEmailError, DomainNotAllowedForRealmError, \ - EmailContainsPlusError -from zproject.backends import email_auth_enabled, email_belongs_to_ldap, check_password_strength - -import logging -import re -import DNS - -from typing import Any, List, Optional, Dict, Tuple -from two_factor.forms import AuthenticationTokenForm as TwoFactorAuthenticationTokenForm -from two_factor.utils import totp_digits +from zerver.models import ( + DisposableEmailError, + DomainNotAllowedForRealmError, + EmailContainsPlusError, + Realm, + UserProfile, + email_to_domain, + get_realm, + get_user_by_delivery_email, +) +from zproject.backends import check_password_strength, email_auth_enabled, email_belongs_to_ldap MIT_VALIDATION_ERROR = 'That user does not exist at MIT or is a ' + \ 'mailing list. ' + \ diff --git a/zerver/lib/actions.py b/zerver/lib/actions.py index 42902cf44b798..6bce33df3a108 100644 --- a/zerver/lib/actions.py +++ b/zerver/lib/actions.py @@ -1,63 +1,116 @@ from typing import ( - AbstractSet, Any, Callable, Dict, Iterable, List, Mapping, MutableMapping, - Optional, Sequence, Set, Tuple, Union, cast, + AbstractSet, + Any, + Callable, + Dict, + Iterable, + List, + Mapping, + MutableMapping, + Optional, + Sequence, + Set, + Tuple, + Union, + cast, ) -from typing_extensions import TypedDict import django.db.utils -from django.db.models import Count, Exists, OuterRef +from django.conf import settings from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ValidationError +from django.core.files import File +from django.db import IntegrityError, connection, transaction +from django.db.models import Count, Exists, F, Max, OuterRef, Q, Sum +from django.db.models.query import QuerySet from django.utils.html import escape +from django.utils.timezone import now as timezone_now from django.utils.translation import ugettext as _ -from django.conf import settings -from django.core.files import File from psycopg2.extras import execute_values from psycopg2.sql import SQL -from analytics.lib.counts import COUNT_STATS, do_increment_logging_stat, \ - RealmCount +from typing_extensions import TypedDict -from zerver.lib.bugdown import ( - version as bugdown_version, +from analytics.lib.counts import COUNT_STATS, RealmCount, do_increment_logging_stat +from analytics.models import StreamCount +from confirmation import settings as confirmation_settings +from confirmation.models import ( + Confirmation, + confirmation_url, + create_confirmation_link, + generate_key, ) +from zerver.decorator import statsd_increment +from zerver.lib import bugdown from zerver.lib.addressee import Addressee -from zerver.lib.bot_config import ( - ConfigError, - get_bot_config, - get_bot_configs, - set_bot_config, +from zerver.lib.alert_words import ( + add_user_alert_words, + get_alert_word_automaton, + remove_user_alert_words, ) +from zerver.lib.avatar import avatar_url, avatar_url_from_dict +from zerver.lib.bot_config import ConfigError, get_bot_config, get_bot_configs, set_bot_config +from zerver.lib.bugdown import version as bugdown_version +from zerver.lib.bulk_create import bulk_create_users from zerver.lib.cache import ( bot_dict_fields, - display_recipient_cache_key, + cache_delete, + cache_delete_many, + cache_set, + cache_set_many, + cache_with_key, delete_user_profile_caches, + display_recipient_cache_key, flush_user_profile, to_dict_cache_key_id, user_profile_by_api_key_cache_key, + user_profile_by_email_cache_key, ) from zerver.lib.context_managers import lockfile +from zerver.lib.create_user import create_user, get_display_email_address from zerver.lib.email_mirror_helpers import encode_email_address, encode_email_address_helper +from zerver.lib.email_notifications import enqueue_welcome_emails +from zerver.lib.email_validation import ( + email_reserved_for_system_bots_error, + get_existing_user_errors, + get_realm_email_validator, + validate_email_is_valid, +) from zerver.lib.emoji import emoji_name_to_emoji_code, get_emoji_file_name -from zerver.lib.exceptions import StreamDoesNotExistError, \ - StreamWithIDDoesNotExistError +from zerver.lib.exceptions import ( + BugdownRenderingException, + ErrorCode, + JsonableError, + StreamDoesNotExistError, + StreamWithIDDoesNotExistError, +) from zerver.lib.export import get_realm_exports_serialized from zerver.lib.external_accounts import DEFAULT_EXTERNAL_ACCOUNTS from zerver.lib.hotspots import get_next_hotspots +from zerver.lib.i18n import get_language_name from zerver.lib.message import ( - access_message, MessageDict, + access_message, render_markdown, - update_first_visible_message_id, truncate_body, truncate_topic, + update_first_visible_message_id, ) from zerver.lib.pysa import mark_sanitized +from zerver.lib.queue import queue_json_publish from zerver.lib.realm_icon import realm_icon_url from zerver.lib.realm_logo import get_realm_logo_data from zerver.lib.retention import move_messages_to_archive -from zerver.lib.send_email import send_email, FromAddress, send_email_to_admins, \ - clear_scheduled_emails, clear_scheduled_invitation_emails +from zerver.lib.send_email import ( + FromAddress, + clear_scheduled_emails, + clear_scheduled_invitation_emails, + send_email, + send_email_to_admins, +) from zerver.lib.server_initialization import create_internal_realm, server_initialized +from zerver.lib.sessions import delete_user_sessions from zerver.lib.storage import static_path +from zerver.lib.stream_recipient import StreamRecipientMap from zerver.lib.stream_subscription import ( get_active_subscriptions_for_stream_id, get_active_subscriptions_for_stream_ids, @@ -68,21 +121,37 @@ num_subscribers_for_stream_id, ) from zerver.lib.stream_topic import StreamTopicTarget +from zerver.lib.streams import ( + access_stream_for_send_message, + check_stream_name, + create_stream_if_needed, + get_default_value_for_history_public_to_subscribers, + render_stream_description, + send_stream_creation_event, + subscribed_to_stream, +) +from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime from zerver.lib.topic import ( + LEGACY_PREV_TOPIC, + ORIG_TOPIC, + TOPIC_LINKS, + TOPIC_NAME, filter_by_exact_message_topic, filter_by_topic_name_via_message, save_message_for_edit_use_case, update_messages_for_topic_edit, - ORIG_TOPIC, - LEGACY_PREV_TOPIC, - TOPIC_LINKS, - TOPIC_NAME, ) -from zerver.lib.topic_mutes import ( - get_topic_mutes, - add_topic_mute, - remove_topic_mute, +from zerver.lib.topic_mutes import add_topic_mute, get_topic_mutes, remove_topic_mute +from zerver.lib.types import ProfileFieldData +from zerver.lib.upload import ( + claim_attachment, + delete_avatar_image, + delete_export_tarball, + delete_message_image, + upload_emoji_image, ) +from zerver.lib.user_groups import access_user_group_by_id, create_user_group +from zerver.lib.user_status import update_user_status from zerver.lib.users import ( check_bot_name_available, check_full_name, @@ -90,93 +159,84 @@ get_api_key, user_profile_to_user_row, ) -from zerver.lib.user_status import ( - update_user_status, -) -from zerver.lib.user_groups import create_user_group, access_user_group_by_id - -from zerver.models import Realm, RealmEmoji, Stream, UserProfile, UserActivity, \ - RealmDomain, Service, SubMessage, \ - Subscription, Recipient, Message, Attachment, UserMessage, RealmAuditLog, \ - UserHotspot, MultiuseInvite, ScheduledMessage, UserStatus, \ - Client, DefaultStream, DefaultStreamGroup, UserPresence, \ - ScheduledEmail, \ - MAX_MESSAGE_LENGTH, get_client, get_stream, \ - get_user_profile_by_id, PreregistrationUser, \ - email_to_username, \ - get_user_by_delivery_email, get_stream_cache_key, active_non_guest_user_ids, \ - UserActivityInterval, active_user_ids, get_active_streams, \ - realm_filters_for_realm, RealmFilter, stream_name_in_use, \ - get_old_unclaimed_attachments, is_cross_realm_bot_email, \ - Reaction, EmailChangeStatus, CustomProfileField, \ - custom_profile_fields_for_realm, get_huddle_user_ids, \ - CustomProfileFieldValue, validate_attachment_request, get_system_bot, \ - query_for_ids, get_huddle_recipient, \ - UserGroup, UserGroupMembership, get_default_stream_groups, \ - get_bot_services, get_bot_dicts_in_realm, \ - get_user_by_id_in_realm_including_cross_realm, \ - get_stream_by_id_in_realm - -from zerver.lib.alert_words import get_alert_word_automaton -from zerver.lib.avatar import avatar_url, avatar_url_from_dict -from zerver.lib.email_validation import get_realm_email_validator, \ - validate_email_is_valid, get_existing_user_errors, \ - email_reserved_for_system_bots_error -from zerver.lib.stream_recipient import StreamRecipientMap +from zerver.lib.utils import generate_api_key, log_statsd_event, statsd from zerver.lib.validator import check_widget_content from zerver.lib.widget import do_widget_post_save_actions - -from django.db import transaction, IntegrityError, connection -from django.db.models import F, Q, Max, Sum -from django.db.models.query import QuerySet -from django.core.exceptions import ValidationError -from django.utils.timezone import now as timezone_now - -from confirmation.models import Confirmation, create_confirmation_link, generate_key, \ - confirmation_url -from confirmation import settings as confirmation_settings - -from zerver.lib.bulk_create import bulk_create_users -from zerver.lib.timestamp import timestamp_to_datetime, datetime_to_timestamp -from zerver.lib.queue import queue_json_publish -from zerver.lib.utils import generate_api_key -from zerver.lib.create_user import create_user, get_display_email_address -from zerver.lib import bugdown -from zerver.lib.cache import cache_with_key, cache_set, \ - user_profile_by_email_cache_key, \ - cache_set_many, cache_delete, cache_delete_many -from zerver.decorator import statsd_increment -from zerver.lib.utils import log_statsd_event, statsd -from zerver.lib.i18n import get_language_name -from zerver.lib.alert_words import add_user_alert_words, \ - remove_user_alert_words -from zerver.lib.email_notifications import enqueue_welcome_emails -from zerver.lib.exceptions import JsonableError, ErrorCode, BugdownRenderingException -from zerver.lib.sessions import delete_user_sessions -from zerver.lib.upload import claim_attachment, delete_message_image, \ - upload_emoji_image, delete_avatar_image, \ - delete_export_tarball +from zerver.models import ( + MAX_MESSAGE_LENGTH, + Attachment, + Client, + CustomProfileField, + CustomProfileFieldValue, + DefaultStream, + DefaultStreamGroup, + EmailChangeStatus, + Message, + MultiuseInvite, + PreregistrationUser, + Reaction, + Realm, + RealmAuditLog, + RealmDomain, + RealmEmoji, + RealmFilter, + Recipient, + ScheduledEmail, + ScheduledMessage, + Service, + Stream, + SubMessage, + Subscription, + UserActivity, + UserActivityInterval, + UserGroup, + UserGroupMembership, + UserHotspot, + UserMessage, + UserPresence, + UserProfile, + UserStatus, + active_non_guest_user_ids, + active_user_ids, + custom_profile_fields_for_realm, + email_to_username, + get_active_streams, + get_bot_dicts_in_realm, + get_bot_services, + get_client, + get_default_stream_groups, + get_huddle_recipient, + get_huddle_user_ids, + get_old_unclaimed_attachments, + get_stream, + get_stream_by_id_in_realm, + get_stream_cache_key, + get_system_bot, + get_user_by_delivery_email, + get_user_by_id_in_realm_including_cross_realm, + get_user_profile_by_id, + is_cross_realm_bot_email, + query_for_ids, + realm_filters_for_realm, + stream_name_in_use, + validate_attachment_request, +) from zerver.tornado.event_queue import send_event -from zerver.lib.types import ProfileFieldData -from zerver.lib.streams import access_stream_for_send_message, subscribed_to_stream, check_stream_name, \ - create_stream_if_needed, get_default_value_for_history_public_to_subscribers, \ - render_stream_description, send_stream_creation_event - -from analytics.models import StreamCount if settings.BILLING_ENABLED: from corporate.lib.stripe import update_license_ledger_if_needed, downgrade_now -import ujson -import time import datetime +import itertools +import logging import os import platform -import logging -import itertools +import time from collections import defaultdict from operator import itemgetter +import ujson + # This will be used to type annotate parameters in a function if the function # works on both str and unicode in python 2 but in python 3 it only works on str. SizedTextIterable = Union[Sequence[str], AbstractSet[str]] diff --git a/zerver/lib/addressee.py b/zerver/lib/addressee.py index cb3dab87896a3..ec1c6fd7a9d9c 100644 --- a/zerver/lib/addressee.py +++ b/zerver/lib/addressee.py @@ -1,15 +1,17 @@ from typing import Iterable, List, Optional, Sequence, Union, cast from django.utils.translation import ugettext as _ + from zerver.lib.exceptions import JsonableError from zerver.models import ( Realm, + Stream, UserProfile, - get_user_including_cross_realm, get_user_by_id_in_realm_including_cross_realm, - Stream, + get_user_including_cross_realm, ) + def get_user_profiles(emails: Iterable[str], realm: Realm) -> List[UserProfile]: user_profiles: List[UserProfile] = [] for email in emails: diff --git a/zerver/lib/alert_words.py b/zerver/lib/alert_words.py index 51c4c49794c5b..e9658d9b971e6 100644 --- a/zerver/lib/alert_words.py +++ b/zerver/lib/alert_words.py @@ -1,10 +1,15 @@ -from django.db import transaction +from typing import Dict, Iterable, List -from zerver.models import UserProfile, Realm, AlertWord, flush_realm_alert_words -from zerver.lib.cache import cache_with_key, realm_alert_words_cache_key, \ - realm_alert_words_automaton_cache_key import ahocorasick -from typing import Dict, Iterable, List +from django.db import transaction + +from zerver.lib.cache import ( + cache_with_key, + realm_alert_words_automaton_cache_key, + realm_alert_words_cache_key, +) +from zerver.models import AlertWord, Realm, UserProfile, flush_realm_alert_words + @cache_with_key(realm_alert_words_cache_key, timeout=3600*24) def alert_words_in_realm(realm: Realm) -> Dict[int, List[str]]: diff --git a/zerver/lib/attachments.py b/zerver/lib/attachments.py index ededc38cbba3d..477770f6586f6 100644 --- a/zerver/lib/attachments.py +++ b/zerver/lib/attachments.py @@ -1,10 +1,12 @@ -from django.utils.translation import ugettext as _ from typing import Any, Dict, List +from django.utils.translation import ugettext as _ + from zerver.lib.request import JsonableError from zerver.lib.upload import delete_message_image from zerver.models import Attachment, UserProfile + def user_attachments(user_profile: UserProfile) -> List[Dict[str, Any]]: attachments = Attachment.objects.filter(owner=user_profile).prefetch_related('messages') return [a.to_dict() for a in attachments] diff --git a/zerver/lib/avatar.py b/zerver/lib/avatar.py index 469b8e27f8489..c6bddfeb45f56 100644 --- a/zerver/lib/avatar.py +++ b/zerver/lib/avatar.py @@ -1,11 +1,16 @@ -from django.conf import settings - +import urllib from typing import Any, Dict, Optional -from zerver.lib.avatar_hash import gravatar_hash, user_avatar_path_from_ids, user_avatar_content_hash -from zerver.lib.upload import upload_backend, MEDIUM_AVATAR_SIZE +from django.conf import settings + +from zerver.lib.avatar_hash import ( + gravatar_hash, + user_avatar_content_hash, + user_avatar_path_from_ids, +) +from zerver.lib.upload import MEDIUM_AVATAR_SIZE, upload_backend from zerver.models import UserProfile -import urllib + def avatar_url(user_profile: UserProfile, medium: bool=False, client_gravatar: bool=False) -> Optional[str]: diff --git a/zerver/lib/avatar_hash.py b/zerver/lib/avatar_hash.py index bbd6c10c17326..492a575cc0db1 100644 --- a/zerver/lib/avatar_hash.py +++ b/zerver/lib/avatar_hash.py @@ -1,10 +1,10 @@ +import hashlib + from django.conf import settings from zerver.lib.utils import make_safe_digest - from zerver.models import UserProfile -import hashlib def gravatar_hash(email: str) -> str: """Compute the Gravatar hash for an email address.""" diff --git a/zerver/lib/bot_config.py b/zerver/lib/bot_config.py index caf6cd20595b9..40e74559cc84d 100644 --- a/zerver/lib/bot_config.py +++ b/zerver/lib/bot_config.py @@ -1,17 +1,16 @@ +import configparser +import importlib +import os +from collections import defaultdict +from typing import Dict, List, Optional + from django.conf import settings from django.db.models import Sum -from django.db.models.query import F from django.db.models.functions import Length -from zerver.models import BotConfigData, UserProfile - -from typing import List, Dict, Optional - -from collections import defaultdict +from django.db.models.query import F -import os +from zerver.models import BotConfigData, UserProfile -import configparser -import importlib class ConfigError(Exception): pass diff --git a/zerver/lib/bot_lib.py b/zerver/lib/bot_lib.py index 492dccd9cfeec..b25b96f5211e8 100644 --- a/zerver/lib/bot_lib.py +++ b/zerver/lib/bot_lib.py @@ -1,23 +1,31 @@ +import importlib import json import os -import importlib -from zerver.lib.actions import internal_send_private_message, \ - internal_send_stream_message_by_name, internal_send_huddle_message -from zerver.models import UserProfile, get_active_user -from zerver.lib.bot_storage import get_bot_storage, set_bot_storage, \ - is_key_in_bot_storage, remove_bot_storage -from zerver.lib.bot_config import get_bot_config, ConfigError -from zerver.lib.integrations import EMBEDDED_BOTS -from zerver.lib.topic import get_topic_from_message_info +from typing import Any, Dict from django.utils.translation import ugettext as _ -from typing import Any, Dict +from zerver.lib.actions import ( + internal_send_huddle_message, + internal_send_private_message, + internal_send_stream_message_by_name, +) +from zerver.lib.bot_config import ConfigError, get_bot_config +from zerver.lib.bot_storage import ( + get_bot_storage, + is_key_in_bot_storage, + remove_bot_storage, + set_bot_storage, +) +from zerver.lib.integrations import EMBEDDED_BOTS +from zerver.lib.topic import get_topic_from_message_info +from zerver.models import UserProfile, get_active_user our_dir = os.path.dirname(os.path.abspath(__file__)) from zulip_bots.lib import RateLimit + def get_bot_handler(service_name: str) -> Any: # Check that this service is present in EMBEDDED_BOTS, add exception handling. diff --git a/zerver/lib/bot_storage.py b/zerver/lib/bot_storage.py index ba3a9b1af3cee..1b43644f0bf27 100644 --- a/zerver/lib/bot_storage.py +++ b/zerver/lib/bot_storage.py @@ -1,10 +1,12 @@ +from typing import List, Optional, Tuple + from django.conf import settings from django.db.models import Sum -from django.db.models.query import F from django.db.models.functions import Length +from django.db.models.query import F + from zerver.models import BotStorageData, UserProfile -from typing import Optional, List, Tuple class StateError(Exception): pass diff --git a/zerver/lib/bugdown/__init__.py b/zerver/lib/bugdown/__init__.py index bfab78823a015..a0bcdc85ccbd0 100644 --- a/zerver/lib/bugdown/__init__.py +++ b/zerver/lib/bugdown/__init__.py @@ -1,65 +1,76 @@ # Zulip's main markdown implementation. See docs/subsystems/markdown.md for # detailed documentation on our markdown syntax. -from typing import (Any, Callable, Dict, Generic, Iterable, List, - Optional, Set, Tuple, TypeVar, Union) -from typing.re import Match, Pattern -from typing_extensions import TypedDict - -import markdown +import functools +import html import logging +import os +import re +import time import traceback import urllib import urllib.parse -import re -import os -import html -import time -import functools +from collections import defaultdict, deque from dataclasses import dataclass -from io import StringIO -import dateutil.parser -import dateutil.tz from datetime import datetime -import xml.etree.ElementTree as etree +from io import StringIO +from typing import ( + Any, + Callable, + Dict, + Generic, + Iterable, + List, + Optional, + Set, + Tuple, + TypeVar, + Union, +) +from typing.re import Match, Pattern +from xml.etree import ElementTree as etree from xml.etree.ElementTree import Element, SubElement -import ahocorasick -from hyperlink import parse - -from collections import deque, defaultdict +import ahocorasick +import dateutil.parser +import dateutil.tz +import markdown import requests - from django.conf import settings from django.db.models import Q +from hyperlink import parse +from markdown.extensions import codehilite, nl2br, sane_lists, tables +from typing_extensions import TypedDict -from markdown.extensions import codehilite, nl2br, tables, sane_lists +from zerver.lib import mention as mention from zerver.lib.bugdown import fenced_code from zerver.lib.bugdown.fenced_code import FENCE_RE +from zerver.lib.cache import NotFoundInCache, cache_with_key from zerver.lib.camo import get_camo_url -from zerver.lib.emoji import translate_emoticons, emoticon_regex, \ - name_to_codepoint, codepoint_to_name -from zerver.lib.mention import possible_mentions, \ - possible_user_group_mentions, extract_user_group -from zerver.lib.url_encoding import encode_stream, hash_util_encode +from zerver.lib.emoji import ( + codepoint_to_name, + emoticon_regex, + name_to_codepoint, + translate_emoticons, +) +from zerver.lib.exceptions import BugdownRenderingException +from zerver.lib.mention import extract_user_group, possible_mentions, possible_user_group_mentions +from zerver.lib.tex import render_tex from zerver.lib.thumbnail import user_uploads_or_external -from zerver.lib.timeout import timeout, TimeoutExpired -from zerver.lib.cache import cache_with_key, NotFoundInCache +from zerver.lib.timeout import TimeoutExpired, timeout +from zerver.lib.timezone import get_common_timezones +from zerver.lib.url_encoding import encode_stream, hash_util_encode from zerver.lib.url_preview import preview as link_preview from zerver.models import ( - all_realm_filters, - get_active_streams, MAX_MESSAGE_LENGTH, Message, Realm, - realm_filters_for_realm, - UserProfile, UserGroup, UserGroupMembership, + UserProfile, + all_realm_filters, + get_active_streams, + realm_filters_for_realm, ) -import zerver.lib.mention as mention -from zerver.lib.tex import render_tex -from zerver.lib.exceptions import BugdownRenderingException -from zerver.lib.timezone import get_common_timezones ReturnT = TypeVar('ReturnT') diff --git a/zerver/lib/bugdown/api_arguments_table_generator.py b/zerver/lib/bugdown/api_arguments_table_generator.py index a9209f7b0511f..5aab6760080f0 100644 --- a/zerver/lib/bugdown/api_arguments_table_generator.py +++ b/zerver/lib/bugdown/api_arguments_table_generator.py @@ -1,13 +1,14 @@ -import re -import os import json +import os +import re +from typing import Any, Dict, List, Optional +import markdown from django.utils.html import escape as escape_html from markdown.extensions import Extension from markdown.preprocessors import Preprocessor + from zerver.openapi.openapi import get_openapi_parameters -from typing import Any, Dict, Optional, List -import markdown REGEXP = re.compile(r'\{generate_api_arguments_table\|\s*(.+?)\s*\|\s*(.+)\s*\}') diff --git a/zerver/lib/bugdown/api_return_values_table_generator.py b/zerver/lib/bugdown/api_return_values_table_generator.py index 865b74006f239..9c56ac3790c34 100644 --- a/zerver/lib/bugdown/api_return_values_table_generator.py +++ b/zerver/lib/bugdown/api_return_values_table_generator.py @@ -1,10 +1,11 @@ import re +from typing import Any, Dict, List, Optional +import markdown from markdown.extensions import Extension from markdown.preprocessors import Preprocessor + from zerver.openapi.openapi import get_openapi_return_values -from typing import Any, Dict, Optional, List -import markdown REGEXP = re.compile(r'\{generate_return_values_table\|\s*(.+?)\s*\|\s*(.+)\s*\}') diff --git a/zerver/lib/bugdown/fenced_code.py b/zerver/lib/bugdown/fenced_code.py index 6eb45bb1db7c0..fe35b6a5f6c28 100644 --- a/zerver/lib/bugdown/fenced_code.py +++ b/zerver/lib/bugdown/fenced_code.py @@ -75,14 +75,15 @@ * [Pygments (optional)](http://pygments.org) """ - import re +from typing import Any, Dict, Iterable, List, MutableSequence, Optional + import markdown from django.utils.html import escape from markdown.extensions.codehilite import CodeHilite, CodeHiliteExtension + from zerver.lib.exceptions import BugdownRenderingException from zerver.lib.tex import render_tex -from typing import Any, Dict, Iterable, List, MutableSequence, Optional # Global vars FENCE_RE = re.compile(""" diff --git a/zerver/lib/bugdown/help_emoticon_translations_table.py b/zerver/lib/bugdown/help_emoticon_translations_table.py index 26e84e5157cdb..079933c6b715c 100644 --- a/zerver/lib/bugdown/help_emoticon_translations_table.py +++ b/zerver/lib/bugdown/help_emoticon_translations_table.py @@ -1,7 +1,8 @@ import re -import markdown from typing import Any, Dict, List from typing.re import Match + +import markdown from markdown.preprocessors import Preprocessor from zerver.lib.emoji import EMOTICON_CONVERSIONS, name_to_codepoint diff --git a/zerver/lib/bugdown/help_relative_links.py b/zerver/lib/bugdown/help_relative_links.py index 910d908471d66..1f086b7cab129 100644 --- a/zerver/lib/bugdown/help_relative_links.py +++ b/zerver/lib/bugdown/help_relative_links.py @@ -1,7 +1,8 @@ import re -import markdown from typing import Any, Dict, List, Optional from typing.re import Match + +import markdown from markdown.preprocessors import Preprocessor # There is a lot of duplicated code between this file and diff --git a/zerver/lib/bugdown/help_settings_links.py b/zerver/lib/bugdown/help_settings_links.py index 4c7b9e34e34d6..f60cccb11696b 100644 --- a/zerver/lib/bugdown/help_settings_links.py +++ b/zerver/lib/bugdown/help_settings_links.py @@ -1,7 +1,8 @@ import re -import markdown from typing import Any, Dict, List, Optional from typing.re import Match + +import markdown from markdown.preprocessors import Preprocessor # There is a lot of duplicated code between this file and diff --git a/zerver/lib/bugdown/include.py b/zerver/lib/bugdown/include.py index 0d3ba9938af10..00536f56c9c04 100644 --- a/zerver/lib/bugdown/include.py +++ b/zerver/lib/bugdown/include.py @@ -1,9 +1,9 @@ -import re import os +import re from typing import Any, Dict, List import markdown -from markdown_include.include import MarkdownInclude, IncludePreprocessor +from markdown_include.include import IncludePreprocessor, MarkdownInclude from zerver.lib.exceptions import InvalidMarkdownIncludeStatement diff --git a/zerver/lib/bugdown/nested_code_blocks.py b/zerver/lib/bugdown/nested_code_blocks.py index 92a457e6c2e90..02f16dd05da38 100644 --- a/zerver/lib/bugdown/nested_code_blocks.py +++ b/zerver/lib/bugdown/nested_code_blocks.py @@ -1,9 +1,11 @@ -from markdown.extensions import Extension -from typing import Any, Dict, Optional, List, Tuple -import markdown +from typing import Any, Dict, List, Optional, Tuple from xml.etree.ElementTree import Element, SubElement -from zerver.lib.bugdown import walk_tree_with_family, ResultWithFamily +import markdown +from markdown.extensions import Extension + +from zerver.lib.bugdown import ResultWithFamily, walk_tree_with_family + class NestedCodeBlocksRenderer(Extension): def extendMarkdown(self, md: markdown.Markdown, md_globals: Dict[str, Any]) -> None: diff --git a/zerver/lib/bugdown/tabbed_sections.py b/zerver/lib/bugdown/tabbed_sections.py index bd38d4e9ed20f..3cea656852753 100644 --- a/zerver/lib/bugdown/tabbed_sections.py +++ b/zerver/lib/bugdown/tabbed_sections.py @@ -1,9 +1,9 @@ import re +from typing import Any, Dict, List, Optional +import markdown from markdown.extensions import Extension from markdown.preprocessors import Preprocessor -from typing import Any, Dict, Optional, List -import markdown START_TABBED_SECTION_REGEX = re.compile(r'^\{start_tabs\}$') END_TABBED_SECTION_REGEX = re.compile(r'^\{end_tabs\}$') diff --git a/zerver/lib/bugdown/testing_mocks.py b/zerver/lib/bugdown/testing_mocks.py index f47e9dd57a0aa..04677c36d636b 100644 --- a/zerver/lib/bugdown/testing_mocks.py +++ b/zerver/lib/bugdown/testing_mocks.py @@ -1,6 +1,6 @@ from typing import Any, Dict, Optional -import ujson +import ujson NORMAL_TWEET = """{ "created_at": "Sat Sep 10 22:23:38 +0000 2011", diff --git a/zerver/lib/bulk_create.py b/zerver/lib/bulk_create.py index 5d05681bccaee..c9881978271e2 100644 --- a/zerver/lib/bulk_create.py +++ b/zerver/lib/bulk_create.py @@ -1,13 +1,12 @@ -from django.db.models import Model - from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union +from django.db.models import Model + +from zerver.lib.create_user import create_user_profile, get_display_email_address from zerver.lib.initial_password import initial_password -from zerver.models import Realm, Stream, UserProfile, \ - Subscription, Recipient, RealmAuditLog -from zerver.lib.create_user import create_user_profile, \ - get_display_email_address from zerver.lib.streams import render_stream_description +from zerver.models import Realm, RealmAuditLog, Recipient, Stream, Subscription, UserProfile + def bulk_create_users(realm: Realm, users_raw: Set[Tuple[str, str, str, bool]], diff --git a/zerver/lib/cache.py b/zerver/lib/cache.py index facb9056fceaa..ad0a7bb3206f3 100644 --- a/zerver/lib/cache.py +++ b/zerver/lib/cache.py @@ -1,32 +1,41 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/caching.html for docs +import base64 +import hashlib +import logging +import os +import random +import re +import sys +import time +import traceback from functools import wraps +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Sequence, + Tuple, + TypeVar, +) -from django.utils.lru_cache import lru_cache +from django.conf import settings from django.core.cache import cache as djcache from django.core.cache import caches -from django.conf import settings -from django.db.models import Q from django.core.cache.backends.base import BaseCache +from django.db.models import Q from django.http import HttpRequest +from django.utils.lru_cache import lru_cache -from typing import Any, Callable, Dict, Iterable, List, \ - Optional, Sequence, TypeVar, Tuple, TYPE_CHECKING - -from zerver.lib.utils import statsd, statsd_key, make_safe_digest -import time -import base64 -import logging -import random -import re -import sys -import traceback -import os -import hashlib +from zerver.lib.utils import make_safe_digest, statsd, statsd_key if TYPE_CHECKING: # These modules have to be imported for type annotations but # they cannot be imported at runtime due to cyclic dependency. - from zerver.models import UserProfile, Realm, Message + from zerver.models import Message, Realm, UserProfile MEMCACHED_MAX_KEY_LENGTH = 250 diff --git a/zerver/lib/cache_helpers.py b/zerver/lib/cache_helpers.py index 944b5c4c27828..fe3526c25cabc 100644 --- a/zerver/lib/cache_helpers.py +++ b/zerver/lib/cache_helpers.py @@ -1,28 +1,38 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/caching.html for docs - -from typing import Any, Callable, Dict, List, Tuple - import datetime import logging +from importlib import import_module +from typing import Any, Callable, Dict, List, Tuple + +from django.conf import settings +from django.contrib.sessions.models import Session +from django.db.models import Q +from django.utils.timezone import now as timezone_now # This file needs to be different from cache.py because cache.py # cannot import anything from zerver.models or we'd have an import # loop from analytics.models import RealmCount -from django.conf import settings -from zerver.models import Message, UserProfile, Stream, get_stream_cache_key, \ - Client, get_client_cache_key, \ - Huddle, huddle_hash_cache_key -from zerver.lib.cache import \ - user_profile_by_api_key_cache_key, \ - user_profile_cache_key, get_remote_cache_time, get_remote_cache_requests, \ - cache_set_many, to_dict_cache_key_id +from zerver.lib.cache import ( + cache_set_many, + get_remote_cache_requests, + get_remote_cache_time, + to_dict_cache_key_id, + user_profile_by_api_key_cache_key, + user_profile_cache_key, +) from zerver.lib.message import MessageDict from zerver.lib.users import get_all_api_keys -from importlib import import_module -from django.contrib.sessions.models import Session -from django.db.models import Q -from django.utils.timezone import now as timezone_now +from zerver.models import ( + Client, + Huddle, + Message, + Stream, + UserProfile, + get_client_cache_key, + get_stream_cache_key, + huddle_hash_cache_key, +) MESSAGE_CACHE_SIZE = 75000 diff --git a/zerver/lib/camo.py b/zerver/lib/camo.py index d1c7a66f1c1e2..5b57daa1201e4 100644 --- a/zerver/lib/camo.py +++ b/zerver/lib/camo.py @@ -1,8 +1,10 @@ -from django.conf import settings import binascii import hashlib import hmac +from django.conf import settings + + def generate_camo_url(url: str) -> str: encoded_url = url.encode("utf-8") encoded_camo_key = settings.CAMO_KEY.encode("utf-8") diff --git a/zerver/lib/ccache.py b/zerver/lib/ccache.py index c3c746dec0e58..a2f159c3cf3ee 100644 --- a/zerver/lib/ccache.py +++ b/zerver/lib/ccache.py @@ -1,4 +1,6 @@ -from typing import Any, Dict, List, Optional +import base64 +import struct +from typing import Any, Dict, List, Optional, Union # This file is adapted from samples/shellinabox/ssh-krb-wrapper in # https://github.com/davidben/webathena, which has the following @@ -26,9 +28,6 @@ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -import base64 -import struct -from typing import Union def force_bytes(s: Union[str, bytes], encoding: str='utf-8') -> bytes: """converts a string to binary string""" diff --git a/zerver/lib/context_managers.py b/zerver/lib/context_managers.py index 1a6700726c650..0ca6c81af6d4f 100644 --- a/zerver/lib/context_managers.py +++ b/zerver/lib/context_managers.py @@ -1,11 +1,10 @@ """ Context managers, i.e. things you can use with the 'with' statement. """ - - import fcntl from contextlib import contextmanager -from typing import Iterator, IO, Any, Union +from typing import IO, Any, Iterator, Union + @contextmanager def flock(lockfile: Union[int, IO[Any]], shared: bool=False) -> Iterator[None]: diff --git a/zerver/lib/create_user.py b/zerver/lib/create_user.py index 7924db0bbb1d2..dc8e16a7718fa 100644 --- a/zerver/lib/create_user.py +++ b/zerver/lib/create_user.py @@ -1,14 +1,22 @@ +from typing import Optional + +import ujson from django.contrib.auth.models import UserManager from django.utils.timezone import now as timezone_now -from zerver.models import UserProfile, Recipient, Subscription, Realm, Stream, \ - PreregistrationUser, get_fake_email_domain -from zerver.lib.upload import copy_avatar + from zerver.lib.hotspots import copy_hotpots +from zerver.lib.upload import copy_avatar from zerver.lib.utils import generate_api_key +from zerver.models import ( + PreregistrationUser, + Realm, + Recipient, + Stream, + Subscription, + UserProfile, + get_fake_email_domain, +) -import ujson - -from typing import Optional def copy_user_settings(source_profile: UserProfile, target_profile: UserProfile) -> None: """Warning: Does not save, to avoid extra database queries""" diff --git a/zerver/lib/db.py b/zerver/lib/db.py index 2971ac2edc44f..32d3d89d7423b 100644 --- a/zerver/lib/db.py +++ b/zerver/lib/db.py @@ -1,9 +1,8 @@ import time -from psycopg2.extensions import cursor, connection -from psycopg2.sql import Composable +from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, TypeVar, Union -from typing import Callable, Optional, Iterable, Any, Dict, List, Union, TypeVar, \ - Mapping, Sequence +from psycopg2.extensions import connection, cursor +from psycopg2.sql import Composable CursorObj = TypeVar('CursorObj', bound=cursor) Query = Union[str, Composable] diff --git a/zerver/lib/debug.py b/zerver/lib/debug.py index d248816db155f..3b44bd52617df 100644 --- a/zerver/lib/debug.py +++ b/zerver/lib/debug.py @@ -8,10 +8,10 @@ import traceback import tracemalloc from types import FrameType +from typing import Optional from django.conf import settings from django.utils.timezone import now as timezone_now -from typing import Optional logger = logging.getLogger('zulip.debug') diff --git a/zerver/lib/dev_ldap_directory.py b/zerver/lib/dev_ldap_directory.py index 2b29cab5e8b27..f703cad52db07 100644 --- a/zerver/lib/dev_ldap_directory.py +++ b/zerver/lib/dev_ldap_directory.py @@ -4,6 +4,7 @@ from typing import Any, Dict, List, Optional from django.conf import settings + from zerver.lib.storage import static_path # See https://jackstromberg.com/2013/01/useraccountcontrol-attributeflag-values/ @@ -59,6 +60,7 @@ def init_fakeldap(directory: Optional[Dict[str, Dict[str, List[str]]]]=None) -> # because its dependency pbr uses pkgresources, which is # really slow to import.) from unittest import mock + from fakeldap import MockLDAP # Silent `django_auth_ldap` logger in dev mode to avoid diff --git a/zerver/lib/digest.py b/zerver/lib/digest.py index 4d7efe49153c1..feadb985de55f 100644 --- a/zerver/lib/digest.py +++ b/zerver/lib/digest.py @@ -1,21 +1,29 @@ -from typing import Any, Dict, List, Set, Tuple, Union - -from collections import defaultdict import datetime import logging +from collections import defaultdict +from typing import Any, Dict, List, Set, Tuple, Union from django.conf import settings from django.utils.timezone import now as timezone_now from confirmation.models import one_click_unsubscribe_link -from zerver.lib.email_notifications import build_message_list -from zerver.lib.send_email import send_future_email, FromAddress -from zerver.lib.url_encoding import encode_stream -from zerver.models import UserProfile, Recipient, Subscription, UserActivity, \ - get_active_streams, get_user_profile_by_id, Realm, Message, RealmAuditLog from zerver.context_processors import common_context -from zerver.lib.queue import queue_json_publish +from zerver.lib.email_notifications import build_message_list from zerver.lib.logging_util import log_to_file +from zerver.lib.queue import queue_json_publish +from zerver.lib.send_email import FromAddress, send_future_email +from zerver.lib.url_encoding import encode_stream +from zerver.models import ( + Message, + Realm, + RealmAuditLog, + Recipient, + Subscription, + UserActivity, + UserProfile, + get_active_streams, + get_user_profile_by_id, +) logger = logging.getLogger(__name__) log_to_file(logger, settings.DIGEST_LOG_PATH) diff --git a/zerver/lib/display_recipient.py b/zerver/lib/display_recipient.py index cea7d78708e83..2b933245cd5b4 100644 --- a/zerver/lib/display_recipient.py +++ b/zerver/lib/display_recipient.py @@ -1,9 +1,14 @@ from typing import Dict, List, Optional, Set, Tuple + from typing_extensions import TypedDict -from zerver.lib.types import DisplayRecipientT, UserDisplayRecipient -from zerver.lib.cache import cache_with_key, display_recipient_cache_key, generic_bulk_cached_fetch, \ - display_recipient_bulk_get_users_by_id_cache_key +from zerver.lib.cache import ( + cache_with_key, + display_recipient_bulk_get_users_by_id_cache_key, + display_recipient_cache_key, + generic_bulk_cached_fetch, +) +from zerver.lib.types import DisplayRecipientT, UserDisplayRecipient from zerver.models import Recipient, Stream, UserProfile, bulk_get_huddle_user_ids display_recipient_fields = [ diff --git a/zerver/lib/domains.py b/zerver/lib/domains.py index 307dc7d4eb5de..570d886efd314 100644 --- a/zerver/lib/domains.py +++ b/zerver/lib/domains.py @@ -1,9 +1,9 @@ +import re +from typing import Optional + from django.core.exceptions import ValidationError from django.utils.translation import ugettext as _ -from typing import Optional - -import re def validate_domain(domain: Optional[str]) -> None: if domain is None or len(domain) == 0: diff --git a/zerver/lib/email_mirror.py b/zerver/lib/email_mirror.py index 59ef84b2149e2..6e1ddd6d5680d 100644 --- a/zerver/lib/email_mirror.py +++ b/zerver/lib/email_mirror.py @@ -1,31 +1,44 @@ -from typing import Dict, Optional, Tuple, List - import logging import re - from email.header import decode_header, make_header from email.message import Message as EmailMessage from email.utils import getaddresses +from typing import Dict, List, Optional, Tuple from django.conf import settings -from django.utils.timezone import timedelta, now as timezone_now - -from zerver.lib.actions import internal_send_private_message, \ - internal_send_stream_message, internal_send_huddle_message -from zerver.lib.email_mirror_helpers import decode_email_address, \ - get_email_gateway_message_string_from_address, ZulipEmailForwardError +from django.utils.timezone import now as timezone_now +from django.utils.timezone import timedelta + +from zerver.lib.actions import ( + internal_send_huddle_message, + internal_send_private_message, + internal_send_stream_message, +) +from zerver.lib.email_mirror_helpers import ( + ZulipEmailForwardError, + decode_email_address, + get_email_gateway_message_string_from_address, +) from zerver.lib.email_notifications import convert_html_to_markdown -from zerver.lib.queue import queue_json_publish -from zerver.lib.utils import generate_random_token -from zerver.lib.upload import upload_message_file -from zerver.lib.send_email import FromAddress -from zerver.lib.rate_limiter import RateLimitedObject from zerver.lib.exceptions import RateLimited from zerver.lib.message import truncate_body, truncate_topic -from zerver.models import Stream, Recipient, MissedMessageEmailAddress, \ - get_display_recipient, \ - Message, Realm, UserProfile, get_system_bot, get_user, get_stream_by_id_in_realm - +from zerver.lib.queue import queue_json_publish +from zerver.lib.rate_limiter import RateLimitedObject +from zerver.lib.send_email import FromAddress +from zerver.lib.upload import upload_message_file +from zerver.lib.utils import generate_random_token +from zerver.models import ( + Message, + MissedMessageEmailAddress, + Realm, + Recipient, + Stream, + UserProfile, + get_display_recipient, + get_stream_by_id_in_realm, + get_system_bot, + get_user, +) from zproject.backends import is_user_active logger = logging.getLogger(__name__) diff --git a/zerver/lib/email_mirror_helpers.py b/zerver/lib/email_mirror_helpers.py index f79ae02ac8813..59ed695385fe3 100644 --- a/zerver/lib/email_mirror_helpers.py +++ b/zerver/lib/email_mirror_helpers.py @@ -1,11 +1,11 @@ import re +from typing import Any, Callable, Dict, Tuple from django.conf import settings from django.utils.text import slugify from zerver.models import Stream -from typing import Any, Callable, Dict, Tuple def default_option_handler_factory(address_option: str) -> Callable[[Dict[str, Any]], None]: def option_setter(options_dict: Dict[str, Any]) -> None: diff --git a/zerver/lib/email_notifications.py b/zerver/lib/email_notifications.py index d7d684078793f..9ba1b0d9ff604 100644 --- a/zerver/lib/email_notifications.py +++ b/zerver/lib/email_notifications.py @@ -1,40 +1,44 @@ +import re +from collections import defaultdict +from datetime import timedelta +from email.utils import formataddr from typing import Any, Dict, Iterable, List, Optional, Tuple -from zerver.lib.types import DisplayRecipientT -from confirmation.models import one_click_unsubscribe_link +import html2text +import lxml.html +import pytz +from bs4 import BeautifulSoup from django.conf import settings +from django.contrib.auth import get_backends from django.utils.timezone import now as timezone_now from django.utils.translation import override as override_language from django.utils.translation import ugettext as _ -from django.contrib.auth import get_backends +from lxml.cssselect import CSSSelector +from confirmation.models import one_click_unsubscribe_link from zerver.decorator import statsd_increment from zerver.lib.message import bulk_access_messages from zerver.lib.queue import queue_json_publish -from zerver.lib.send_email import send_future_email, FromAddress -from zerver.lib.url_encoding import personal_narrow_url, huddle_narrow_url, \ - stream_narrow_url, topic_narrow_url +from zerver.lib.send_email import FromAddress, send_future_email +from zerver.lib.types import DisplayRecipientT +from zerver.lib.url_encoding import ( + huddle_narrow_url, + personal_narrow_url, + stream_narrow_url, + topic_narrow_url, +) from zerver.models import ( + Message, Recipient, - UserMessage, Stream, - get_display_recipient, + UserMessage, UserProfile, + get_context_for_message, + get_display_recipient, get_user_profile_by_id, receives_offline_email_notifications, - get_context_for_message, - Message, ) -from datetime import timedelta -from email.utils import formataddr -import html2text -from lxml.cssselect import CSSSelector -import lxml.html -import re -from collections import defaultdict -import pytz -from bs4 import BeautifulSoup def relative_to_full_url(base_url: str, content: str) -> str: # Convert relative URLs to absolute URLs. @@ -297,6 +301,7 @@ def do_send_missedmessage_events_reply_in_zulip(user_profile: UserProfile, for a group of messages that share a recipient (and topic) """ from zerver.context_processors import common_context + # Disabled missedmessage emails internally if not user_profile.enable_offline_email_notifications: return @@ -558,7 +563,7 @@ def enqueue_welcome_emails(user: UserProfile, realm_creation: bool=False) -> Non context['getting_started_link'] = "https://zulip.com" # Imported here to avoid import cycles. - from zproject.backends import email_belongs_to_ldap, ZulipLDAPAuthBackend + from zproject.backends import ZulipLDAPAuthBackend, email_belongs_to_ldap if email_belongs_to_ldap(user.realm, user.delivery_email): context["ldap"] = True diff --git a/zerver/lib/email_validation.py b/zerver/lib/email_validation.py index d67398fd98830..dc37305518ebc 100644 --- a/zerver/lib/email_validation.py +++ b/zerver/lib/email_validation.py @@ -3,21 +3,22 @@ from django.core import validators from django.core.exceptions import ValidationError from django.utils.translation import ugettext as _ -from zerver.lib.name_restrictions import is_disposable_domain +from zerver.lib.name_restrictions import is_disposable_domain # TODO: Move DisposableEmailError, etc. into here. from zerver.models import ( - email_to_username, - email_to_domain, - get_users_by_delivery_email, - is_cross_realm_bot_email, DisposableEmailError, DomainNotAllowedForRealmError, EmailContainsPlusError, Realm, RealmDomain, + email_to_domain, + email_to_username, + get_users_by_delivery_email, + is_cross_realm_bot_email, ) + def validate_disposable(email: str) -> None: if is_disposable_domain(email_to_domain(email)): raise DisposableEmailError diff --git a/zerver/lib/emoji.py b/zerver/lib/emoji.py index 902732d747fce..df339df2f2e2f 100644 --- a/zerver/lib/emoji.py +++ b/zerver/lib/emoji.py @@ -1,14 +1,14 @@ import os import re -import ujson +from typing import Optional, Tuple +import ujson from django.utils.translation import ugettext as _ -from typing import Optional, Tuple +from zerver.lib.exceptions import OrganizationAdministratorRequired from zerver.lib.request import JsonableError from zerver.lib.storage import static_path from zerver.lib.upload import upload_backend -from zerver.lib.exceptions import OrganizationAdministratorRequired from zerver.models import Reaction, Realm, RealmEmoji, UserProfile emoji_codes_path = static_path("generated/emoji/emoji_codes.json") diff --git a/zerver/lib/error_notify.py b/zerver/lib/error_notify.py index fb3d5ddba54f5..21b8f09960096 100644 --- a/zerver/lib/error_notify.py +++ b/zerver/lib/error_notify.py @@ -1,16 +1,17 @@ # System documented in https://zulip.readthedocs.io/en/latest/subsystems/logging.html - from collections import defaultdict +from typing import Any, Dict, Optional + from django.conf import settings from django.core.mail import mail_admins from django.http import HttpResponse from django.utils.translation import ugettext as _ -from typing import Any, Dict, Optional from zerver.filters import clean_data_from_query_parameters -from zerver.models import get_stream, get_system_bot from zerver.lib.actions import internal_send_stream_message -from zerver.lib.response import json_success, json_error +from zerver.lib.response import json_error, json_success +from zerver.models import get_stream, get_system_bot + def format_email_subject(email_subject: str) -> str: """ diff --git a/zerver/lib/events.py b/zerver/lib/events.py index 1522dd9fcdeae..018b18f55d8fd 100644 --- a/zerver/lib/events.py +++ b/zerver/lib/events.py @@ -1,20 +1,28 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/events-system.html for # high-level documentation on how this system works. - import copy +from importlib import import_module +from typing import Any, Callable, Dict, Iterable, Optional, Sequence, Set -from django.utils.translation import ugettext as _ from django.conf import settings -from importlib import import_module -from typing import ( - Any, Callable, Dict, Iterable, Optional, Sequence, Set, -) +from django.utils.translation import ugettext as _ session_engine = import_module(settings.SESSION_ENGINE) +from version import API_FEATURE_LEVEL, ZULIP_VERSION +from zerver.lib.actions import ( + default_stream_groups_to_dicts_sorted, + do_get_streams, + gather_subscriptions_helper, + get_available_notification_sounds, + get_default_streams_for_realm, + get_owned_bot_dicts, + streams_to_dicts_sorted, +) from zerver.lib.alert_words import user_alert_words from zerver.lib.avatar import avatar_url from zerver.lib.bot_config import load_bot_config_template +from zerver.lib.external_accounts import DEFAULT_EXTERNAL_ACCOUNTS from zerver.lib.hotspots import get_next_hotspots from zerver.lib.integrations import EMBEDDED_BOTS, WEBHOOK_INTEGRATIONS from zerver.lib.message import ( @@ -27,40 +35,34 @@ remove_message_id_from_unread_mgs, ) from zerver.lib.narrow import check_supported_events_narrow_filter, read_stop_words -from zerver.lib.presence import ( - get_presences_for_realm, - get_presence_for_user, -) +from zerver.lib.presence import get_presence_for_user, get_presences_for_realm from zerver.lib.push_notifications import push_notifications_enabled -from zerver.lib.soft_deactivation import reactivate_user_if_soft_deactivated from zerver.lib.realm_icon import realm_icon_url from zerver.lib.realm_logo import get_realm_logo_url from zerver.lib.request import JsonableError +from zerver.lib.soft_deactivation import reactivate_user_if_soft_deactivated from zerver.lib.stream_subscription import handle_stream_notifications_compatibility from zerver.lib.topic import TOPIC_NAME from zerver.lib.topic_mutes import get_topic_mutes -from zerver.lib.actions import ( - do_get_streams, - get_default_streams_for_realm, - gather_subscriptions_helper, - streams_to_dicts_sorted, - default_stream_groups_to_dicts_sorted, - get_owned_bot_dicts, - get_available_notification_sounds, -) -from zerver.lib.users import get_cross_realm_dicts, get_raw_user_data, is_administrator_role from zerver.lib.user_groups import user_groups_in_realm_serialized from zerver.lib.user_status import get_user_info_dict -from zerver.tornado.event_queue import request_event_queue, get_user_events +from zerver.lib.users import get_cross_realm_dicts, get_raw_user_data, is_administrator_role from zerver.models import ( - Client, Message, Realm, UserProfile, UserMessage, + Client, + CustomProfileField, + Message, + Realm, + Stream, + UserMessage, + UserProfile, + custom_profile_fields_for_realm, + get_default_stream_groups, + get_realm_domains, realm_filters_for_realm, - custom_profile_fields_for_realm, get_realm_domains, - get_default_stream_groups, CustomProfileField, Stream, ) +from zerver.tornado.event_queue import get_user_events, request_event_queue from zproject.backends import email_auth_enabled, password_auth_enabled -from version import ZULIP_VERSION, API_FEATURE_LEVEL -from zerver.lib.external_accounts import DEFAULT_EXTERNAL_ACCOUNTS + def add_realm_logo_fields(state: Dict[str, Any], realm: Realm) -> None: state['realm_logo_url'] = get_realm_logo_url(realm, night = False) diff --git a/zerver/lib/exceptions.py b/zerver/lib/exceptions.py index 897c2dca6e130..57db95d943a80 100644 --- a/zerver/lib/exceptions.py +++ b/zerver/lib/exceptions.py @@ -1,9 +1,8 @@ from enum import Enum -from typing import Any, Dict, List, NoReturn, Type, TypeVar, Optional +from typing import Any, Dict, List, NoReturn, Optional, Type, TypeVar from django.utils.translation import ugettext as _ - T = TypeVar("T", bound="AbstractEnum") class AbstractEnum(Enum): diff --git a/zerver/lib/export.py b/zerver/lib/export.py index 7bbe837770a1c..090a40187f026 100644 --- a/zerver/lib/export.py +++ b/zerver/lib/export.py @@ -7,35 +7,61 @@ # (2) if it doesn't belong in EXCLUDED_TABLES, add a Config object for # it to get_realm_config. import datetime +import glob +import logging +import os +import shutil +import subprocess +import tempfile +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union + import boto3 +import ujson from boto3.resources.base import ServiceResource from django.apps import apps from django.conf import settings from django.forms.models import model_to_dict -from django.utils.timezone import make_aware as timezone_make_aware from django.utils.timezone import is_naive as timezone_is_naive -import glob -import logging -import os -import ujson -import subprocess -import tempfile -import shutil +from django.utils.timezone import make_aware as timezone_make_aware + +import zerver.lib.upload +from analytics.models import RealmCount, StreamCount, UserCount from scripts.lib.zulip_tools import overwrite_symlink from zerver.lib.avatar_hash import user_avatar_path_from_ids -from analytics.models import RealmCount, UserCount, StreamCount -from zerver.models import UserProfile, Realm, Client, Huddle, Stream, \ - UserMessage, Subscription, Message, RealmEmoji, RealmFilter, Reaction, \ - RealmDomain, Recipient, DefaultStream, get_user_profile_by_id, \ - UserPresence, UserActivity, UserActivityInterval, CustomProfileField, \ - CustomProfileFieldValue, get_display_recipient, Attachment, get_system_bot, \ - RealmAuditLog, UserHotspot, MutedTopic, Service, UserGroup, \ - UserGroupMembership, BotStorageData, BotConfigData from zerver.lib.pysa import mark_sanitized - -import zerver.lib.upload -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, \ - Union +from zerver.models import ( + Attachment, + BotConfigData, + BotStorageData, + Client, + CustomProfileField, + CustomProfileFieldValue, + DefaultStream, + Huddle, + Message, + MutedTopic, + Reaction, + Realm, + RealmAuditLog, + RealmDomain, + RealmEmoji, + RealmFilter, + Recipient, + Service, + Stream, + Subscription, + UserActivity, + UserActivityInterval, + UserGroup, + UserGroupMembership, + UserHotspot, + UserMessage, + UserPresence, + UserProfile, + get_display_recipient, + get_system_bot, + get_user_profile_by_id, +) # Custom mypy types follow: Record = Dict[str, Any] diff --git a/zerver/lib/external_accounts.py b/zerver/lib/external_accounts.py index bcb08c17c69b5..81a07728956da 100644 --- a/zerver/lib/external_accounts.py +++ b/zerver/lib/external_accounts.py @@ -2,11 +2,15 @@ This module stores data for "External Account" custom profile field. """ from typing import Optional + from django.utils.translation import ugettext as _ -from zerver.lib.validator import check_required_string, \ - check_external_account_url_pattern, check_dict_only from zerver.lib.types import ProfileFieldData +from zerver.lib.validator import ( + check_dict_only, + check_external_account_url_pattern, + check_required_string, +) # Default external account fields are by default available # to realm admins, where realm admin only need to select diff --git a/zerver/lib/fix_unreads.py b/zerver/lib/fix_unreads.py index 9bc46a8f4d69e..f13b2811f7602 100644 --- a/zerver/lib/fix_unreads.py +++ b/zerver/lib/fix_unreads.py @@ -1,7 +1,7 @@ -import time import logging - +import time from typing import Callable, List, TypeVar + from psycopg2.extensions import cursor from psycopg2.sql import SQL diff --git a/zerver/lib/generate_test_data.py b/zerver/lib/generate_test_data.py index 979bd5ad82d00..a1432535abe01 100644 --- a/zerver/lib/generate_test_data.py +++ b/zerver/lib/generate_test_data.py @@ -1,11 +1,13 @@ import itertools -import ujson -import random -from typing import List, Dict, Any import os +import random +from typing import Any, Dict, List + +import ujson from scripts.lib.zulip_tools import get_or_create_dev_uuid_var_path + def load_config() -> Dict[str, Any]: with open("zerver/tests/fixtures/config.generate_data.json") as infile: config = ujson.load(infile) diff --git a/zerver/lib/hotspots.py b/zerver/lib/hotspots.py index c37491f3d37e9..322654d522301 100644 --- a/zerver/lib/hotspots.py +++ b/zerver/lib/hotspots.py @@ -1,11 +1,11 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/hotspots.html # for documentation on this subsystem. +from typing import Dict, List + from django.conf import settings from django.utils.translation import ugettext as _ -from zerver.models import UserProfile, UserHotspot - -from typing import List, Dict +from zerver.models import UserHotspot, UserProfile ALL_HOTSPOTS: Dict[str, Dict[str, str]] = { 'intro_reply': { diff --git a/zerver/lib/html_diff.py b/zerver/lib/html_diff.py index 16686a69e49e9..cc8d4fad6c8ee 100644 --- a/zerver/lib/html_diff.py +++ b/zerver/lib/html_diff.py @@ -1,7 +1,8 @@ -import lxml +from typing import Optional +import lxml from lxml.html.diff import htmldiff -from typing import Optional + def highlight_with_class(text: str, klass: str) -> str: return f'{text}' diff --git a/zerver/lib/html_to_text.py b/zerver/lib/html_to_text.py index c7bac8e2758a2..9a450f5dc8113 100644 --- a/zerver/lib/html_to_text.py +++ b/zerver/lib/html_to_text.py @@ -6,6 +6,7 @@ from zerver.lib.cache import cache_with_key, open_graph_description_cache_key + def html_to_text(content: str, tags: Optional[Dict[str, str]]=None) -> str: bs = BeautifulSoup(content, features='lxml') # Skip any admonition (warning) blocks, since they're diff --git a/zerver/lib/i18n.py b/zerver/lib/i18n.py index b63d4c593429b..8caf89ad9c9d3 100644 --- a/zerver/lib/i18n.py +++ b/zerver/lib/i18n.py @@ -1,16 +1,15 @@ +import logging import operator +import os +from itertools import zip_longest +from typing import Any, Dict, List +import ujson from django.conf import settings from django.utils import translation -from django.utils.translation import ugettext as _ from django.utils.lru_cache import lru_cache +from django.utils.translation import ugettext as _ -from itertools import zip_longest -from typing import Any, List, Dict - -import os -import ujson -import logging def with_language(string: str, language: str) -> str: """ diff --git a/zerver/lib/import_realm.py b/zerver/lib/import_realm.py index c58673002ab6e..ce0f769afb115 100644 --- a/zerver/lib/import_realm.py +++ b/zerver/lib/import_realm.py @@ -1,44 +1,71 @@ import datetime import logging import os -import ujson import shutil +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, cast import boto3 +import ujson from bs4 import BeautifulSoup from django.conf import settings from django.db import connection from django.db.models import Max from django.utils.timezone import now as timezone_now -from typing import Any, Dict, List, Optional, Set, Tuple, \ - Iterable, cast from psycopg2.extras import execute_values -from psycopg2.sql import Identifier, SQL +from psycopg2.sql import SQL, Identifier from analytics.models import RealmCount, StreamCount, UserCount -from zerver.lib.actions import UserMessageLite, bulk_insert_ums, \ - do_change_plan_type, do_change_avatar_fields +from zerver.lib.actions import ( + UserMessageLite, + bulk_insert_ums, + do_change_avatar_fields, + do_change_plan_type, +) from zerver.lib.avatar_hash import user_avatar_path_from_ids +from zerver.lib.bugdown import version as bugdown_version from zerver.lib.bulk_create import bulk_create_users, bulk_set_users_or_streams_recipient_fields -from zerver.lib.timestamp import datetime_to_timestamp -from zerver.lib.export import DATE_FIELDS, \ - Record, TableData, TableName, Field, Path +from zerver.lib.export import DATE_FIELDS, Field, Path, Record, TableData, TableName from zerver.lib.message import do_render_markdown -from zerver.lib.bugdown import version as bugdown_version +from zerver.lib.parallel import run_parallel +from zerver.lib.server_initialization import create_internal_realm, server_initialized from zerver.lib.streams import render_stream_description -from zerver.lib.upload import random_name, sanitize_name, \ - guess_type, BadImageError +from zerver.lib.timestamp import datetime_to_timestamp +from zerver.lib.upload import BadImageError, guess_type, random_name, sanitize_name from zerver.lib.utils import generate_api_key, process_list_in_batches -from zerver.lib.parallel import run_parallel -from zerver.lib.server_initialization import server_initialized, create_internal_realm -from zerver.models import UserProfile, Realm, Client, Huddle, Stream, \ - UserMessage, Subscription, Message, RealmEmoji, \ - RealmDomain, Recipient, get_user_profile_by_id, \ - UserPresence, UserActivity, UserActivityInterval, Reaction, \ - CustomProfileField, CustomProfileFieldValue, RealmAuditLog, \ - Attachment, get_system_bot, email_to_username, get_huddle_hash, \ - UserHotspot, MutedTopic, Service, UserGroup, UserGroupMembership, \ - BotStorageData, BotConfigData, DefaultStream, RealmFilter +from zerver.models import ( + Attachment, + BotConfigData, + BotStorageData, + Client, + CustomProfileField, + CustomProfileFieldValue, + DefaultStream, + Huddle, + Message, + MutedTopic, + Reaction, + Realm, + RealmAuditLog, + RealmDomain, + RealmEmoji, + RealmFilter, + Recipient, + Service, + Stream, + Subscription, + UserActivity, + UserActivityInterval, + UserGroup, + UserGroupMembership, + UserHotspot, + UserMessage, + UserPresence, + UserProfile, + email_to_username, + get_huddle_hash, + get_system_bot, + get_user_profile_by_id, +) realm_tables = [("zerver_defaultstream", DefaultStream, "defaultstream"), ("zerver_realmemoji", RealmEmoji, "realmemoji"), @@ -707,6 +734,7 @@ def import_uploads(realm: Realm, import_dir: Path, processes: int, processing_av if processing_avatars: from zerver.lib.upload import upload_backend + # Ensure that we have medium-size avatar images for every # avatar. TODO: This implementation is hacky, both in that it # does get_user_profile_by_id for each user, and in that it diff --git a/zerver/lib/initial_password.py b/zerver/lib/initial_password.py index 4e2d5360f90fb..e53a088ef1ca9 100644 --- a/zerver/lib/initial_password.py +++ b/zerver/lib/initial_password.py @@ -1,10 +1,9 @@ -from django.conf import settings - -import hashlib import base64 - +import hashlib from typing import Optional +from django.conf import settings + def initial_password(email: str) -> Optional[str]: """Given an email address, returns the initial password for that account, as diff --git a/zerver/lib/integrations.py b/zerver/lib/integrations.py index abadb2db152fb..8a1c81e1fd947 100644 --- a/zerver/lib/integrations.py +++ b/zerver/lib/integrations.py @@ -1,16 +1,16 @@ import os - from dataclasses import dataclass -from typing import Dict, List, Optional, Any, Tuple +from typing import Any, Dict, List, Optional, Tuple + from django.conf.urls import url from django.contrib.staticfiles.storage import staticfiles_storage from django.urls.resolvers import RegexPattern from django.utils.module_loading import import_string from django.utils.translation import ugettext as _ + from zerver.lib.storage import static_path from zerver.lib.types import Validator - """This module declares all of the (documented) integrations available in the Zulip server. The Integration class is used as part of generating the documentation on the /integrations page, while the diff --git a/zerver/lib/logging_util.py b/zerver/lib/logging_util.py index 23ddb43e218f1..e250494fd9c33 100644 --- a/zerver/lib/logging_util.py +++ b/zerver/lib/logging_util.py @@ -1,16 +1,16 @@ # System documented in https://zulip.readthedocs.io/en/latest/subsystems/logging.html - -from django.utils.timezone import now as timezone_now - import hashlib import logging import threading import traceback -from typing import Optional, Tuple from datetime import datetime, timedelta, timezone +from logging import Logger +from typing import Optional, Tuple + from django.conf import settings from django.core.cache import cache -from logging import Logger +from django.utils.timezone import now as timezone_now + class _RateLimitFilter: """This class is designed to rate-limit Django error reporting diff --git a/zerver/lib/management.py b/zerver/lib/management.py index f7c90fa9f5bd9..3da9b47a6e7d4 100644 --- a/zerver/lib/management.py +++ b/zerver/lib/management.py @@ -1,15 +1,14 @@ # Library code for use in management commands - import time - from argparse import ArgumentParser, RawTextHelpFormatter +from typing import Any, Dict, List, Optional from django.conf import settings from django.core.exceptions import MultipleObjectsReturned from django.core.management.base import BaseCommand, CommandError -from typing import Any, Dict, Optional, List -from zerver.models import Realm, UserProfile, Client, get_client +from zerver.models import Client, Realm, UserProfile, get_client + def is_integer_string(val: str) -> bool: try: diff --git a/zerver/lib/mdiff.py b/zerver/lib/mdiff.py index 09a2a93098a7a..3bde46030f889 100755 --- a/zerver/lib/mdiff.py +++ b/zerver/lib/mdiff.py @@ -1,6 +1,7 @@ +import logging import os import subprocess -import logging + class DiffException(Exception): pass diff --git a/zerver/lib/mention.py b/zerver/lib/mention.py index b1fc0e8e9c1be..2691190dc1dde 100644 --- a/zerver/lib/mention.py +++ b/zerver/lib/mention.py @@ -1,6 +1,5 @@ -from typing import Optional, Set, Tuple - import re +from typing import Optional, Set, Tuple # Match multi-word string between @** ** or match any one-word # sequences after @ diff --git a/zerver/lib/message.py b/zerver/lib/message.py index 1d783e39e7f17..5be36afc3715e 100644 --- a/zerver/lib/message.py +++ b/zerver/lib/message.py @@ -1,64 +1,54 @@ +import copy import datetime -import ujson import zlib -import ahocorasick -import copy +from typing import Any, Dict, List, Optional, Sequence, Set, Tuple -from django.utils.translation import ugettext as _ -from django.utils.timezone import now as timezone_now +import ahocorasick +import ujson from django.db import connection from django.db.models import Sum +from django.utils.timezone import now as timezone_now +from django.utils.translation import ugettext as _ from psycopg2.sql import SQL +from typing_extensions import TypedDict from analytics.lib.counts import COUNT_STATS, RealmCount - +from zerver.lib import bugdown as bugdown from zerver.lib.avatar import get_avatar_field -import zerver.lib.bugdown as bugdown from zerver.lib.cache import ( cache_with_key, generic_bulk_cached_fetch, to_dict_cache_key, to_dict_cache_key_id, ) -from zerver.lib.display_recipient import UserDisplayRecipient, DisplayRecipientT, \ - bulk_fetch_display_recipients -from zerver.lib.request import JsonableError -from zerver.lib.stream_subscription import ( - get_stream_subscriptions_for_user, +from zerver.lib.display_recipient import ( + DisplayRecipientT, + UserDisplayRecipient, + bulk_fetch_display_recipients, ) +from zerver.lib.request import JsonableError +from zerver.lib.stream_subscription import get_stream_subscriptions_for_user from zerver.lib.timestamp import datetime_to_timestamp -from zerver.lib.topic import ( - DB_TOPIC_NAME, - MESSAGE__TOPIC, - TOPIC_LINKS, - TOPIC_NAME, -) -from zerver.lib.topic_mutes import ( - build_topic_mute_checker, - topic_is_muted, -) - +from zerver.lib.topic import DB_TOPIC_NAME, MESSAGE__TOPIC, TOPIC_LINKS, TOPIC_NAME +from zerver.lib.topic_mutes import build_topic_mute_checker, topic_is_muted from zerver.models import ( - get_display_recipient_by_id, - get_user_profile_by_id, - query_for_ids, + MAX_MESSAGE_LENGTH, + MAX_TOPIC_NAME_LENGTH, Message, + Reaction, Realm, Recipient, Stream, SubMessage, Subscription, - UserProfile, UserMessage, - Reaction, + UserProfile, + get_display_recipient_by_id, + get_user_profile_by_id, get_usermessage_by_message_id, - MAX_MESSAGE_LENGTH, - MAX_TOPIC_NAME_LENGTH, + query_for_ids, ) -from typing import Any, Dict, List, Optional, Set, Tuple, Sequence -from typing_extensions import TypedDict - RealmAlertWord = Dict[int, List[str]] class RawUnreadMessagesResult(TypedDict): diff --git a/zerver/lib/migrate.py b/zerver/lib/migrate.py index 3fa2bd3167028..8fb53992619bc 100644 --- a/zerver/lib/migrate.py +++ b/zerver/lib/migrate.py @@ -1,8 +1,8 @@ -from psycopg2.extensions import cursor -from psycopg2.sql import Composable, Identifier, SQL +import time from typing import List, TypeVar -import time +from psycopg2.extensions import cursor +from psycopg2.sql import SQL, Composable, Identifier CursorObj = TypeVar('CursorObj', bound=cursor) diff --git a/zerver/lib/mobile_auth_otp.py b/zerver/lib/mobile_auth_otp.py index 83df1fef71aa4..bd009fc92eb1b 100644 --- a/zerver/lib/mobile_auth_otp.py +++ b/zerver/lib/mobile_auth_otp.py @@ -7,10 +7,11 @@ # # The decryption logic here isn't actually used by the flow; we just # have it here as part of testing the overall library. - import binascii + from zerver.models import UserProfile + def xor_hex_strings(bytes_a: str, bytes_b: str) -> str: """Given two hex strings of equal length, return a hex string with the bitwise xor of the two hex strings.""" diff --git a/zerver/lib/name_restrictions.py b/zerver/lib/name_restrictions.py index 2db6d57ceb23c..02f3f9f401d2a 100644 --- a/zerver/lib/name_restrictions.py +++ b/zerver/lib/name_restrictions.py @@ -1,5 +1,6 @@ from disposable_email_domains import blacklist + def is_reserved_subdomain(subdomain: str) -> bool: if subdomain in ZULIP_RESERVED_SUBDOMAINS: return True diff --git a/zerver/lib/narrow.py b/zerver/lib/narrow.py index d3c080a8e5adf..e77f18913039b 100644 --- a/zerver/lib/narrow.py +++ b/zerver/lib/narrow.py @@ -1,13 +1,11 @@ import os +from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence -from zerver.lib.request import JsonableError -from zerver.lib.topic import ( - get_topic_from_message_info, -) from django.conf import settings from django.utils.translation import ugettext as _ -from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence +from zerver.lib.request import JsonableError +from zerver.lib.topic import get_topic_from_message_info stop_words_list: Optional[List[str]] = None def read_stop_words() -> List[str]: diff --git a/zerver/lib/onboarding.py b/zerver/lib/onboarding.py index bcdde3ac5ae6f..9b8b593a2d59e 100644 --- a/zerver/lib/onboarding.py +++ b/zerver/lib/onboarding.py @@ -1,15 +1,19 @@ +from typing import Dict, List + from django.conf import settings from django.db.models import Count from django.utils.translation import ugettext as _ -from zerver.lib.actions import \ - internal_prep_stream_message_by_name, internal_send_private_message, \ - do_send_messages, \ - do_add_reaction, create_users +from zerver.lib.actions import ( + create_users, + do_add_reaction, + do_send_messages, + internal_prep_stream_message_by_name, + internal_send_private_message, +) from zerver.lib.emoji import emoji_name_to_emoji_code from zerver.models import Message, Realm, UserProfile, get_system_bot -from typing import Dict, List def missing_any_realm_internal_bots() -> bool: bot_emails = [bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN,) diff --git a/zerver/lib/outgoing_webhook.py b/zerver/lib/outgoing_webhook.py index 88f2e62255e21..bfcc7e2ac99f1 100644 --- a/zerver/lib/outgoing_webhook.py +++ b/zerver/lib/outgoing_webhook.py @@ -1,23 +1,29 @@ -from typing import Any, AnyStr, Dict, Optional - import abc -import requests import json import logging -from requests import Response +from typing import Any, AnyStr, Dict, Optional +import requests from django.utils.translation import ugettext as _ +from requests import Response -from zerver.models import UserProfile, get_user_profile_by_id, get_client, \ - GENERIC_INTERFACE, Service, SLACK_INTERFACE, email_to_domain +from version import ZULIP_VERSION +from zerver.decorator import JsonableError from zerver.lib.actions import check_send_message from zerver.lib.message import MessageDict from zerver.lib.queue import retry_event from zerver.lib.topic import get_topic_from_message_info from zerver.lib.url_encoding import near_message_url -from zerver.decorator import JsonableError +from zerver.models import ( + GENERIC_INTERFACE, + SLACK_INTERFACE, + Service, + UserProfile, + email_to_domain, + get_client, + get_user_profile_by_id, +) -from version import ZULIP_VERSION class OutgoingWebhookServiceInterface(metaclass=abc.ABCMeta): diff --git a/zerver/lib/parallel.py b/zerver/lib/parallel.py index 91765874bf044..b182b73aae428 100644 --- a/zerver/lib/parallel.py +++ b/zerver/lib/parallel.py @@ -1,9 +1,8 @@ -from typing import Dict, Iterable, Tuple, Callable, TypeVar, Iterator - +import errno import os import pty import sys -import errno +from typing import Callable, Dict, Iterable, Iterator, Tuple, TypeVar JobData = TypeVar('JobData') diff --git a/zerver/lib/presence.py b/zerver/lib/presence.py index 58c99a343d95d..425153f628116 100644 --- a/zerver/lib/presence.py +++ b/zerver/lib/presence.py @@ -1,21 +1,14 @@ -from collections import defaultdict - import datetime import itertools import time +from collections import defaultdict +from typing import Any, Dict, List, Set from django.utils.timezone import now as timezone_now -from typing import Any, Dict, List, Set - from zerver.lib.timestamp import datetime_to_timestamp -from zerver.models import ( - query_for_ids, - PushDeviceToken, - Realm, - UserPresence, - UserProfile, -) +from zerver.models import PushDeviceToken, Realm, UserPresence, UserProfile, query_for_ids + def get_status_dicts_for_rows(all_rows: List[Dict[str, Any]], mobile_user_ids: Set[int], diff --git a/zerver/lib/profile.py b/zerver/lib/profile.py index 200a434e5d608..3aac3844be919 100644 --- a/zerver/lib/profile.py +++ b/zerver/lib/profile.py @@ -1,7 +1,6 @@ import cProfile - from functools import wraps -from typing import Any, TypeVar, Callable +from typing import Any, Callable, TypeVar ReturnT = TypeVar('ReturnT') diff --git a/zerver/lib/push_notifications.py b/zerver/lib/push_notifications.py index 5abbdd370a587..1029eab664d12 100644 --- a/zerver/lib/push_notifications.py +++ b/zerver/lib/push_notifications.py @@ -1,32 +1,37 @@ import base64 import binascii import logging -import lxml.html import re import time +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union -from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING, Union - +import gcm +import lxml.html +import ujson from django.conf import settings from django.db import IntegrityError, transaction from django.db.models import F from django.utils.timezone import now as timezone_now from django.utils.translation import ugettext as _ -import gcm -import ujson from zerver.decorator import statsd_increment from zerver.lib.avatar import absolute_avatar_url from zerver.lib.exceptions import JsonableError -from zerver.lib.message import access_message, \ - bulk_access_messages_expect_usermessage, huddle_users -from zerver.lib.remote_server import send_to_push_bouncer, send_json_to_push_bouncer +from zerver.lib.message import access_message, bulk_access_messages_expect_usermessage, huddle_users +from zerver.lib.remote_server import send_json_to_push_bouncer, send_to_push_bouncer from zerver.lib.timestamp import datetime_to_timestamp -from zerver.models import PushDeviceToken, Message, Recipient, \ - UserMessage, UserProfile, \ - get_display_recipient, receives_offline_push_notifications, \ - receives_online_notifications, get_user_profile_by_id, \ - ArchivedMessage +from zerver.models import ( + ArchivedMessage, + Message, + PushDeviceToken, + Recipient, + UserMessage, + UserProfile, + get_display_recipient, + get_user_profile_by_id, + receives_offline_push_notifications, + receives_online_notifications, +) if TYPE_CHECKING: from apns2.client import APNsClient diff --git a/zerver/lib/pysa.py b/zerver/lib/pysa.py index 6019b29378d48..c0fa12484a2a4 100644 --- a/zerver/lib/pysa.py +++ b/zerver/lib/pysa.py @@ -1,6 +1,5 @@ from typing import TypeVar - T = TypeVar("T") diff --git a/zerver/lib/queue.py b/zerver/lib/queue.py index 176a06ac02bef..f132e2a11a30b 100644 --- a/zerver/lib/queue.py +++ b/zerver/lib/queue.py @@ -1,17 +1,17 @@ -from collections import defaultdict import logging import random import threading import time +from collections import defaultdict from typing import Any, Callable, Dict, List, Mapping, Optional, Set -from django.conf import settings import pika import pika.adapters.tornado_connection +import ujson +from django.conf import settings from pika.adapters.blocking_connection import BlockingChannel from pika.spec import Basic from tornado import ioloop -import ujson from zerver.lib.utils import statsd diff --git a/zerver/lib/rate_limiter.py b/zerver/lib/rate_limiter.py index 87c7fbf944536..78826dc48d03f 100644 --- a/zerver/lib/rate_limiter.py +++ b/zerver/lib/rate_limiter.py @@ -1,20 +1,18 @@ +import logging import os - +import time from abc import ABC, abstractmethod from typing import Dict, List, Optional, Tuple, Type +import redis from django.conf import settings from django.http import HttpRequest + from zerver.lib.exceptions import RateLimited from zerver.lib.redis_utils import get_redis_client from zerver.lib.utils import statsd - from zerver.models import UserProfile -import logging -import redis -import time - # Implement a rate-limiting scheme inspired by the one described here, but heavily modified # https://www.domaintools.com/resources/blog/rate-limiting-with-redis diff --git a/zerver/lib/realm_description.py b/zerver/lib/realm_description.py index 39b10e91568f5..409a4d3fbd9bf 100644 --- a/zerver/lib/realm_description.py +++ b/zerver/lib/realm_description.py @@ -1,8 +1,12 @@ -from zerver.models import Realm -from zerver.lib.cache import cache_with_key, realm_rendered_description_cache_key, \ - realm_text_description_cache_key from zerver.lib.bugdown import convert as bugdown_convert +from zerver.lib.cache import ( + cache_with_key, + realm_rendered_description_cache_key, + realm_text_description_cache_key, +) from zerver.lib.html_to_text import html_to_text +from zerver.models import Realm + @cache_with_key(realm_rendered_description_cache_key, timeout=3600*24*7) def get_realm_rendered_description(realm: Realm) -> str: diff --git a/zerver/lib/realm_icon.py b/zerver/lib/realm_icon.py index 81a9eeb1d141d..3813d08048733 100644 --- a/zerver/lib/realm_icon.py +++ b/zerver/lib/realm_icon.py @@ -4,6 +4,7 @@ from zerver.lib.upload import upload_backend from zerver.models import Realm + def realm_icon_url(realm: Realm) -> str: return get_realm_icon_url(realm) diff --git a/zerver/lib/realm_logo.py b/zerver/lib/realm_logo.py index 6c3eac83ea2f1..a4273ac805662 100644 --- a/zerver/lib/realm_logo.py +++ b/zerver/lib/realm_logo.py @@ -5,6 +5,7 @@ from zerver.lib.upload import upload_backend from zerver.models import Realm + def get_realm_logo_url(realm: Realm, night: bool) -> str: if night: logo_source = realm.night_logo_source diff --git a/zerver/lib/redis_utils.py b/zerver/lib/redis_utils.py index 11144e16276d9..af150680bb56e 100644 --- a/zerver/lib/redis_utils.py +++ b/zerver/lib/redis_utils.py @@ -1,10 +1,11 @@ -from django.conf import settings +import re from typing import Any, Dict, Optional -from zerver.lib.utils import generate_random_token -import re import redis import ujson +from django.conf import settings + +from zerver.lib.utils import generate_random_token # Redis accepts keys up to 512MB in size, but there's no reason for us to use such size, # so we want to stay limited to 1024 characters. diff --git a/zerver/lib/remote_server.py b/zerver/lib/remote_server.py index 892c249b79877..7ed113fe44d0b 100644 --- a/zerver/lib/remote_server.py +++ b/zerver/lib/remote_server.py @@ -1,9 +1,9 @@ import logging -import requests -import ujson import urllib from typing import Any, Dict, List, Optional, Tuple, Union +import requests +import ujson from django.conf import settings from django.forms.models import model_to_dict from django.utils.translation import ugettext as _ @@ -14,6 +14,7 @@ from zerver.lib.export import floatify_datetime_fields from zerver.models import RealmAuditLog + class PushNotificationBouncerException(Exception): pass diff --git a/zerver/lib/request.py b/zerver/lib/request.py index 515c420998d42..d542445b57204 100644 --- a/zerver/lib/request.py +++ b/zerver/lib/request.py @@ -1,18 +1,16 @@ from collections import defaultdict from functools import wraps from types import FunctionType -import ujson +from typing import Any, Callable, Dict, Generic, List, Optional, TypeVar, Union, cast, overload +import ujson +from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ +from typing_extensions import Literal -from zerver.lib.exceptions import JsonableError, ErrorCode, \ - InvalidJSONError +from zerver.lib.exceptions import ErrorCode, InvalidJSONError, JsonableError from zerver.lib.types import Validator, ViewFuncT -from django.http import HttpRequest, HttpResponse - -from typing import Any, Callable, Dict, Generic, List, Optional, TypeVar, Union, cast, overload -from typing_extensions import Literal class RequestConfusingParmsError(JsonableError): code = ErrorCode.REQUEST_CONFUSING_VAR diff --git a/zerver/lib/response.py b/zerver/lib/response.py index 52e1576401a9e..a9e5516a76c6b 100644 --- a/zerver/lib/response.py +++ b/zerver/lib/response.py @@ -1,10 +1,12 @@ +from typing import Any, Dict, List, Optional + +import ujson from django.http import HttpResponse, HttpResponseNotAllowed from django.utils.translation import ugettext as _ -import ujson -from typing import Optional, Any, Dict, List from zerver.lib.exceptions import JsonableError + class HttpResponseUnauthorized(HttpResponse): status_code = 401 diff --git a/zerver/lib/rest.py b/zerver/lib/rest.py index 3954b4aa05358..1430ac21dfe7a 100644 --- a/zerver/lib/rest.py +++ b/zerver/lib/rest.py @@ -1,16 +1,20 @@ from functools import wraps from typing import Any, Callable, Dict -from django.utils.module_loading import import_string +from django.conf import settings +from django.http import HttpRequest, HttpResponse, HttpResponseRedirect from django.utils.cache import add_never_cache_headers +from django.utils.module_loading import import_string from django.views.decorators.csrf import csrf_exempt, csrf_protect -from zerver.decorator import authenticated_json_view, authenticated_rest_api_view, \ - process_as_post, authenticated_uploads_api_view, \ - ReturnT +from zerver.decorator import ( + ReturnT, + authenticated_json_view, + authenticated_rest_api_view, + authenticated_uploads_api_view, + process_as_post, +) from zerver.lib.response import json_method_not_allowed, json_unauthorized -from django.http import HttpRequest, HttpResponse, HttpResponseRedirect -from django.conf import settings METHODS = ('GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'PATCH') FLAGS = ('override_api_url_scheme') diff --git a/zerver/lib/retention.py b/zerver/lib/retention.py index f23a03319c20f..8320c56ac78c0 100644 --- a/zerver/lib/retention.py +++ b/zerver/lib/retention.py @@ -1,20 +1,30 @@ +import logging from datetime import timedelta +from typing import Any, Dict, List, Optional, Tuple from django.conf import settings from django.db import connection, transaction from django.db.models import Model from django.utils.timezone import now as timezone_now -from psycopg2.sql import Composable, Identifier, Literal, SQL +from psycopg2.sql import SQL, Composable, Identifier, Literal from zerver.lib.logging_util import log_to_file -from zerver.models import (Message, UserMessage, ArchivedUserMessage, Realm, - Attachment, ArchivedAttachment, Reaction, ArchivedReaction, - SubMessage, ArchivedSubMessage, Recipient, Stream, ArchiveTransaction, - get_user_including_cross_realm) - -from typing import Any, Dict, List, Optional, Tuple - -import logging +from zerver.models import ( + ArchivedAttachment, + ArchivedReaction, + ArchivedSubMessage, + ArchivedUserMessage, + ArchiveTransaction, + Attachment, + Message, + Reaction, + Realm, + Recipient, + Stream, + SubMessage, + UserMessage, + get_user_including_cross_realm, +) logger = logging.getLogger('zulip.retention') log_to_file(logger, settings.RETENTION_LOG_PATH) diff --git a/zerver/lib/send_email.py b/zerver/lib/send_email.py index 955786c9f2aa8..ee9a523ed989b 100644 --- a/zerver/lib/send_email.py +++ b/zerver/lib/send_email.py @@ -1,30 +1,26 @@ +import datetime +import hashlib +import logging +import os +from email.parser import Parser +from email.policy import default +from email.utils import formataddr, parseaddr +from typing import Any, Dict, List, Mapping, Optional, Tuple + +import ujson from django.conf import settings from django.core.mail import EmailMultiAlternatives from django.core.management import CommandError from django.template import loader +from django.template.exceptions import TemplateDoesNotExist from django.utils.timezone import now as timezone_now from django.utils.translation import override as override_language from django.utils.translation import ugettext as _ -from django.template.exceptions import TemplateDoesNotExist -from scripts.setup.inline_email_css import inline_template - -from zerver.models import ScheduledEmail, get_user_profile_by_id, \ - EMAIL_TYPES, Realm, UserProfile -import datetime -from email.utils import parseaddr, formataddr -from email.parser import Parser -from email.policy import default - -import logging -import ujson -import hashlib - -import os -from typing import Any, Dict, List, Mapping, Optional, Tuple - -from zerver.lib.logging_util import log_to_file from confirmation.models import generate_key +from scripts.setup.inline_email_css import inline_template +from zerver.lib.logging_util import log_to_file +from zerver.models import EMAIL_TYPES, Realm, ScheduledEmail, UserProfile, get_user_profile_by_id ## Logging setup ## diff --git a/zerver/lib/server_initialization.py b/zerver/lib/server_initialization.py index cf74b46cb2ee7..cdf01857608d6 100644 --- a/zerver/lib/server_initialization.py +++ b/zerver/lib/server_initialization.py @@ -1,10 +1,10 @@ +from typing import Iterable, Optional, Tuple + from django.conf import settings from zerver.lib.bulk_create import bulk_create_users -from zerver.models import Realm, UserProfile, email_to_username, get_client, \ - get_system_bot +from zerver.models import Realm, UserProfile, email_to_username, get_client, get_system_bot -from typing import Iterable, Optional, Tuple def server_initialized() -> bool: return Realm.objects.exists() diff --git a/zerver/lib/sessions.py b/zerver/lib/sessions.py index d61a4cbecd2f1..63422aadcb81c 100644 --- a/zerver/lib/sessions.py +++ b/zerver/lib/sessions.py @@ -1,15 +1,15 @@ import logging - from datetime import timedelta +from importlib import import_module +from typing import Any, List, Mapping, Optional + from django.conf import settings from django.contrib.auth import SESSION_KEY, get_user_model from django.contrib.sessions.models import Session from django.utils.timezone import now as timezone_now -from importlib import import_module -from typing import Any, List, Mapping, Optional -from zerver.models import Realm, UserProfile, get_user_profile_by_id from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime +from zerver.models import Realm, UserProfile, get_user_profile_by_id session_engine = import_module(settings.SESSION_ENGINE) diff --git a/zerver/lib/soft_deactivation.py b/zerver/lib/soft_deactivation.py index bd0e2d1fbebaf..f6f2aa45e38cd 100644 --- a/zerver/lib/soft_deactivation.py +++ b/zerver/lib/soft_deactivation.py @@ -1,16 +1,24 @@ # Documented in https://zulip.readthedocs.io/en/latest/subsystems/sending-messages.html#soft-deactivation - -from zerver.lib.logging_util import log_to_file -from collections import defaultdict import logging +from collections import defaultdict +from typing import Any, DefaultDict, Dict, List, Optional, Union + +from django.conf import settings from django.db import transaction from django.db.models import Max -from django.conf import settings from django.utils.timezone import now as timezone_now -from typing import DefaultDict, Dict, List, Optional, Union, Any -from zerver.models import UserProfile, UserMessage, RealmAuditLog, \ - Subscription, Message, Recipient, UserActivity, Realm +from zerver.lib.logging_util import log_to_file +from zerver.models import ( + Message, + Realm, + RealmAuditLog, + Recipient, + Subscription, + UserActivity, + UserMessage, + UserProfile, +) logger = logging.getLogger("zulip.soft_deactivation") log_to_file(logger, settings.SOFT_DEACTIVATION_LOG_PATH) diff --git a/zerver/lib/sqlalchemy_utils.py b/zerver/lib/sqlalchemy_utils.py index 1770f70c092c1..c6314bbf8ff8d 100644 --- a/zerver/lib/sqlalchemy_utils.py +++ b/zerver/lib/sqlalchemy_utils.py @@ -1,9 +1,10 @@ -from typing import Optional, Any +from typing import Any, Optional +import sqlalchemy from django.db import connection + from zerver.lib.db import TimeTrackingConnection -import sqlalchemy # This is a Pool that doesn't close connections. Therefore it can be used with # existing Django database connections. diff --git a/zerver/lib/statistics.py b/zerver/lib/statistics.py index 72d4f47356368..7b41508e2d221 100644 --- a/zerver/lib/statistics.py +++ b/zerver/lib/statistics.py @@ -1,7 +1,8 @@ -from zerver.models import UserProfile, UserActivityInterval - from datetime import datetime, timedelta +from zerver.models import UserActivityInterval, UserProfile + + # Return the amount of Zulip usage for this user between the two # given dates def seconds_usage_between(user_profile: UserProfile, begin: datetime, end: datetime) -> timedelta: diff --git a/zerver/lib/stream_recipient.py b/zerver/lib/stream_recipient.py index d25cab5cb3da1..7d96d7b90b9a3 100644 --- a/zerver/lib/stream_recipient.py +++ b/zerver/lib/stream_recipient.py @@ -1,9 +1,11 @@ -from typing import (Dict, List) +from typing import Dict, List from django.db import connection from psycopg2.sql import SQL + from zerver.models import Recipient + class StreamRecipientMap: ''' This class maps stream_id -> recipient_id and vice versa. diff --git a/zerver/lib/stream_subscription.py b/zerver/lib/stream_subscription.py index dc9b6d2c0be08..2185083fbdf86 100644 --- a/zerver/lib/stream_subscription.py +++ b/zerver/lib/stream_subscription.py @@ -1,12 +1,9 @@ from typing import Any, Dict, List, Tuple from django.db.models.query import QuerySet -from zerver.models import ( - Recipient, - Stream, - Subscription, - UserProfile, -) + +from zerver.models import Recipient, Stream, Subscription, UserProfile + def get_active_subscriptions_for_stream_id(stream_id: int) -> QuerySet: # TODO: Change return type to QuerySet[Subscription] diff --git a/zerver/lib/stream_topic.py b/zerver/lib/stream_topic.py index d3bacf1a1e15a..e5fd1d4c6405b 100644 --- a/zerver/lib/stream_topic.py +++ b/zerver/lib/stream_topic.py @@ -1,13 +1,10 @@ from typing import Set + from django.db.models.query import QuerySet -from zerver.lib.stream_subscription import ( - get_active_subscriptions_for_stream_id, -) +from zerver.lib.stream_subscription import get_active_subscriptions_for_stream_id +from zerver.models import MutedTopic -from zerver.models import ( - MutedTopic, -) class StreamTopicTarget: ''' diff --git a/zerver/lib/streams.py b/zerver/lib/streams.py index 3f7158ddf931f..cf96a5d9c5b49 100644 --- a/zerver/lib/streams.py +++ b/zerver/lib/streams.py @@ -1,19 +1,27 @@ -from typing import Any, Iterable, List, Mapping, Set, Tuple, Optional, Union +from typing import Any, Iterable, List, Mapping, Optional, Set, Tuple, Union -from django.utils.translation import ugettext as _ from django.conf import settings +from django.db.models.query import QuerySet +from django.utils.translation import ugettext as _ +from zerver.lib.bugdown import convert as bugdown_convert from zerver.lib.request import JsonableError from zerver.models import ( - UserProfile, Stream, Subscription, Realm, Recipient, get_stream, - bulk_get_streams, get_realm_stream, DefaultStreamGroup, get_stream_by_id_in_realm, - is_cross_realm_bot_email, active_non_guest_user_ids, + DefaultStreamGroup, + Realm, + Recipient, + Stream, + Subscription, + UserProfile, + active_non_guest_user_ids, + bulk_get_streams, + get_realm_stream, + get_stream, + get_stream_by_id_in_realm, + is_cross_realm_bot_email, ) -from zerver.lib.bugdown import convert as bugdown_convert from zerver.tornado.event_queue import send_event -from django.db.models.query import QuerySet - def get_default_value_for_history_public_to_subscribers( realm: Realm, diff --git a/zerver/lib/subdomains.py b/zerver/lib/subdomains.py index 77800dc1aaffe..824c95937db2c 100644 --- a/zerver/lib/subdomains.py +++ b/zerver/lib/subdomains.py @@ -1,10 +1,12 @@ -from django.conf import settings -from django.http import HttpRequest import re from typing import Optional +from django.conf import settings +from django.http import HttpRequest + from zerver.models import Realm, UserProfile + def get_subdomain(request: HttpRequest) -> str: # The HTTP spec allows, but doesn't require, a client to omit the diff --git a/zerver/lib/test_classes.py b/zerver/lib/test_classes.py index d96ad05dcb0e7..de686cd6dead1 100644 --- a/zerver/lib/test_classes.py +++ b/zerver/lib/test_classes.py @@ -1,59 +1,49 @@ +import base64 +import os +import re +import shutil +import tempfile +import urllib from contextlib import contextmanager from email.utils import parseaddr -from fakeldap import MockLDAP -from typing import (cast, Any, Dict, Iterable, - Iterator, List, Optional, - Tuple, Union, Set) +from typing import Any, Dict, Iterable, Iterator, List, Optional, Set, Tuple, Union, cast +from unittest import mock +import ujson from django.apps import apps -from django.db.migrations.state import StateApps -from django.urls import resolve from django.conf import settings -from django.test import TestCase -from django.test.client import ( - BOUNDARY, MULTIPART_CONTENT, encode_multipart, -) -from django.test.testcases import SerializeMixin -from django.http import HttpResponse -from django.db.migrations.executor import MigrationExecutor from django.db import connection +from django.db.migrations.executor import MigrationExecutor +from django.db.migrations.state import StateApps from django.db.utils import IntegrityError -from django.http import HttpRequest +from django.http import HttpRequest, HttpResponse +from django.test import TestCase +from django.test.client import BOUNDARY, MULTIPART_CONTENT, encode_multipart +from django.test.testcases import SerializeMixin +from django.urls import resolve from django.utils import translation - +from fakeldap import MockLDAP from two_factor.models import PhoneDevice -from zerver.lib.initial_password import initial_password -from zerver.lib.users import get_api_key -from zerver.lib.sessions import get_session_dict_user -from zerver.lib.webhooks.common import get_fixture_http_headers, standardize_headers +from zerver.decorator import do_two_factor_login from zerver.lib.actions import ( - check_send_message, bulk_add_subscriptions, + bulk_add_subscriptions, bulk_remove_subscriptions, - check_send_stream_message, gather_subscriptions, + check_send_message, + check_send_stream_message, + gather_subscriptions, ) +from zerver.lib.initial_password import initial_password +from zerver.lib.sessions import get_session_dict_user +from zerver.lib.stream_subscription import get_stream_subscriptions_for_user from zerver.lib.streams import ( create_stream_if_needed, get_default_value_for_history_public_to_subscribers, ) -from zerver.lib.stream_subscription import ( - get_stream_subscriptions_for_user, -) - -from zerver.lib.test_helpers import ( - instrument_url, find_key_by_email, -) - +from zerver.lib.test_helpers import find_key_by_email, instrument_url +from zerver.lib.users import get_api_key +from zerver.lib.webhooks.common import get_fixture_http_headers, standardize_headers from zerver.models import ( - clear_supported_auth_backends_cache, - flush_per_request_caches, - get_stream, - get_client, - get_display_recipient, - get_user, - get_user_by_delivery_email, - get_realm, - get_system_bot, Client, Message, Realm, @@ -61,20 +51,20 @@ Stream, Subscription, UserProfile, + clear_supported_auth_backends_cache, + flush_per_request_caches, + get_client, + get_display_recipient, + get_realm, get_realm_stream, + get_stream, + get_system_bot, + get_user, + get_user_by_delivery_email, ) -from zilencer.models import get_remote_server_by_uuid -from zerver.decorator import do_two_factor_login from zerver.tornado.event_queue import clear_client_event_queues_for_testing +from zilencer.models import get_remote_server_by_uuid -import base64 -from unittest import mock -import os -import re -import ujson -import urllib -import shutil -import tempfile class UploadSerializeMixin(SerializeMixin): """ diff --git a/zerver/lib/test_fixtures.py b/zerver/lib/test_fixtures.py index b038dc627f167..8d918aa0c537a 100644 --- a/zerver/lib/test_fixtures.py +++ b/zerver/lib/test_fixtures.py @@ -1,27 +1,29 @@ +import glob import json import os import re +import shutil import subprocess import sys -from typing import Any, List, Set +import time from importlib import import_module from io import StringIO -import glob -import time -import shutil +from typing import Any, List, Set -from django.db import connections, DEFAULT_DB_ALIAS, ProgrammingError, \ - connection -from django.db.utils import OperationalError from django.apps import apps from django.conf import settings from django.core.management import call_command +from django.db import DEFAULT_DB_ALIAS, ProgrammingError, connection, connections +from django.db.utils import OperationalError from django.utils.module_loading import module_has_submodule sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) from scripts.lib.zulip_tools import ( - get_dev_uuid_var_path, run, TEMPLATE_DATABASE_DIR, - is_digest_obsolete, write_new_digest, + TEMPLATE_DATABASE_DIR, + get_dev_uuid_var_path, + is_digest_obsolete, + run, + write_new_digest, ) UUID_VAR_DIR = get_dev_uuid_var_path() diff --git a/zerver/lib/test_helpers.py b/zerver/lib/test_helpers.py index 3f0bcb302a98b..d1d7c9ef123a9 100644 --- a/zerver/lib/test_helpers.py +++ b/zerver/lib/test_helpers.py @@ -1,57 +1,68 @@ from contextlib import contextmanager from typing import ( - Any, Callable, Dict, Generator, Iterable, Iterator, List, Mapping, - Optional, Tuple, Union, IO, TypeVar, TYPE_CHECKING, + IO, + TYPE_CHECKING, + Any, + Callable, + Dict, + Generator, + Iterable, + Iterator, + List, + Mapping, + Optional, + Tuple, + TypeVar, + Union, ) -from django.urls import URLResolver -from django.conf import settings -from django.test import override_settings -from django.http import HttpResponse, HttpResponseRedirect -from django.db.migrations.state import StateApps import boto3 from boto3.resources.base import ServiceResource +from django.conf import settings +from django.db.migrations.state import StateApps +from django.http import HttpResponse, HttpResponseRedirect +from django.test import override_settings +from django.urls import URLResolver import zerver.lib.upload +from zerver.lib import cache from zerver.lib.actions import do_set_realm_property -from zerver.lib.upload import S3UploadBackend, LocalUploadBackend from zerver.lib.avatar import avatar_url from zerver.lib.cache import get_cache_backend from zerver.lib.db import Params, ParamsT, Query, TimeTrackingCursor -from zerver.lib import cache -from zerver.tornado import event_queue -from zerver.tornado.handlers import allocate_handler_id -from zerver.worker import queue_processors from zerver.lib.integrations import WEBHOOK_INTEGRATIONS - +from zerver.lib.upload import LocalUploadBackend, S3UploadBackend from zerver.models import ( - get_realm, - get_stream, Client, Message, Realm, Subscription, UserMessage, UserProfile, + get_realm, + get_stream, ) - -from zproject.backends import ExternalAuthResult, ExternalAuthDataDict +from zerver.tornado import event_queue +from zerver.tornado.handlers import allocate_handler_id +from zerver.worker import queue_processors +from zproject.backends import ExternalAuthDataDict, ExternalAuthResult if TYPE_CHECKING: # Avoid an import cycle; we only need these for type annotations. from zerver.lib.test_classes import ZulipTestCase, MigrationsTestCase import collections -from unittest import mock import os import re import sys import time -import ujson -from moto import mock_s3 +from unittest import mock import fakeldap import ldap +import ujson +from moto import mock_s3 + class MockLDAP(fakeldap.MockLDAP): class LDAPError(ldap.LDAPError): diff --git a/zerver/lib/test_runner.py b/zerver/lib/test_runner.py index e01590441a467..faa2527876fb2 100644 --- a/zerver/lib/test_runner.py +++ b/zerver/lib/test_runner.py @@ -1,38 +1,33 @@ -from functools import partial +import os import random +import shutil import sys - -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, \ - Type, TypeVar, Union +import time +import unittest +from functools import partial +from multiprocessing.sharedctypes import Synchronized +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, TypeVar, Union from unittest import loader, runner from unittest.result import TestResult from django.conf import settings -from django.db import connections, ProgrammingError -from django.urls.resolvers import URLPattern +from django.db import ProgrammingError, connections from django.test import TestCase from django.test import runner as django_runner from django.test.runner import DiscoverRunner from django.test.signals import template_rendered +from django.urls.resolvers import URLPattern +from scripts.lib.zulip_tools import ( + TEMPLATE_DATABASE_DIR, + get_dev_uuid_var_path, + get_or_create_dev_uuid_var_path, +) from zerver.lib import test_helpers from zerver.lib.cache import bounce_key_prefix_for_testing from zerver.lib.rate_limiter import bounce_redis_key_prefix_for_testing from zerver.lib.sqlalchemy_utils import get_sqlalchemy_connection -from zerver.lib.test_helpers import ( - write_instrumentation_reports, - append_instrumentation_data, -) - -import os -import time -import unittest -import shutil - -from multiprocessing.sharedctypes import Synchronized - -from scripts.lib.zulip_tools import get_dev_uuid_var_path, TEMPLATE_DATABASE_DIR, \ - get_or_create_dev_uuid_var_path +from zerver.lib.test_helpers import append_instrumentation_data, write_instrumentation_reports # We need to pick an ID for this test-backend invocation, and store it # in this global so it can be used in init_worker; this is used to diff --git a/zerver/lib/tex.py b/zerver/lib/tex.py index 429f2d6d7a591..a53f2d4d16004 100644 --- a/zerver/lib/tex.py +++ b/zerver/lib/tex.py @@ -1,10 +1,13 @@ import logging import os import subprocess -from django.conf import settings from typing import Optional + +from django.conf import settings + from zerver.lib.storage import static_path + def render_tex(tex: str, is_inline: bool=True) -> Optional[str]: r"""Render a TeX string into HTML using KaTeX diff --git a/zerver/lib/thumbnail.py b/zerver/lib/thumbnail.py index 7897a9d5fcada..115589a208fea 100644 --- a/zerver/lib/thumbnail.py +++ b/zerver/lib/thumbnail.py @@ -4,6 +4,7 @@ import sys import urllib from urllib.parse import urljoin + from django.conf import settings from django.utils.http import is_safe_url from libthumbor import CryptoURL @@ -11,10 +12,9 @@ ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) -from zthumbor.loaders.helpers import ( - THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE, THUMBOR_EXTERNAL_TYPE, -) from zerver.lib.camo import get_camo_url +from zthumbor.loaders.helpers import THUMBOR_EXTERNAL_TYPE, THUMBOR_LOCAL_FILE_TYPE, THUMBOR_S3_TYPE + def is_thumbor_enabled() -> bool: return settings.THUMBOR_URL != '' diff --git a/zerver/lib/timeout.py b/zerver/lib/timeout.py index 84418df695f00..043afe4c9a639 100644 --- a/zerver/lib/timeout.py +++ b/zerver/lib/timeout.py @@ -1,10 +1,9 @@ -from types import TracebackType -from typing import Any, Callable, Optional, Tuple, Type, TypeVar - -import sys -import time import ctypes +import sys import threading +import time +from types import TracebackType +from typing import Any, Callable, Optional, Tuple, Type, TypeVar # Based on https://code.activestate.com/recipes/483752/ diff --git a/zerver/lib/timestamp.py b/zerver/lib/timestamp.py index d344bef6f528e..dcbc3dee499e4 100644 --- a/zerver/lib/timestamp.py +++ b/zerver/lib/timestamp.py @@ -1,5 +1,6 @@ -import datetime import calendar +import datetime + class TimezoneNotUTCException(Exception): pass diff --git a/zerver/lib/timezone.py b/zerver/lib/timezone.py index 949a98568297b..ac88347c6f117 100644 --- a/zerver/lib/timezone.py +++ b/zerver/lib/timezone.py @@ -1,7 +1,8 @@ -from typing import List, Dict, Union, Any +import datetime +from typing import Any, Dict, List, Union import pytz -import datetime + def get_all_timezones() -> List[str]: return sorted(pytz.all_timezones) diff --git a/zerver/lib/topic.py b/zerver/lib/topic.py index b5098f37398a9..55fe2b504c1d8 100644 --- a/zerver/lib/topic.py +++ b/zerver/lib/topic.py @@ -1,22 +1,11 @@ -from django.db import connection -from django.db.models.query import QuerySet, Q +from typing import Any, Dict, List, Optional, Tuple -from sqlalchemy.sql import ( - column, - literal, - func, -) +from django.db import connection +from django.db.models.query import Q, QuerySet +from sqlalchemy.sql import column, func, literal from zerver.lib.request import REQ -from zerver.models import ( - Message, - Recipient, - Stream, - UserMessage, - UserProfile, -) - -from typing import Any, Dict, List, Optional, Tuple +from zerver.models import Message, Recipient, Stream, UserMessage, UserProfile # Only use these constants for events. ORIG_TOPIC = "orig_subject" diff --git a/zerver/lib/topic_mutes.py b/zerver/lib/topic_mutes.py index 359dbcf0d9123..f1a30df3237ba 100644 --- a/zerver/lib/topic_mutes.py +++ b/zerver/lib/topic_mutes.py @@ -1,24 +1,13 @@ -from typing import Any, Callable, Dict, List, Optional, Union import datetime +from typing import Any, Callable, Dict, List, Optional, Union + +from django.utils.timezone import now as timezone_now +from sqlalchemy.sql import Selectable, and_, column, not_, or_ -from zerver.lib.topic import ( - topic_match_sa, -) from zerver.lib.timestamp import datetime_to_timestamp -from zerver.models import ( - get_stream, - MutedTopic, - UserProfile, -) -from sqlalchemy.sql import ( - and_, - column, - not_, - or_, - Selectable, -) +from zerver.lib.topic import topic_match_sa +from zerver.models import MutedTopic, UserProfile, get_stream -from django.utils.timezone import now as timezone_now def get_topic_mutes(user_profile: UserProfile) -> List[List[Union[str, float]]]: rows = MutedTopic.objects.filter( diff --git a/zerver/lib/transfer.py b/zerver/lib/transfer.py index cf077279936bc..5656240b61b92 100644 --- a/zerver/lib/transfer.py +++ b/zerver/lib/transfer.py @@ -1,14 +1,14 @@ -import os import logging +import os +from mimetypes import guess_type from django.conf import settings from django.db import connection -from mimetypes import guess_type -from zerver.models import UserProfile, Attachment, RealmEmoji from zerver.lib.avatar_hash import user_avatar_path -from zerver.lib.upload import S3UploadBackend, upload_image_to_s3 from zerver.lib.parallel import run_parallel +from zerver.lib.upload import S3UploadBackend, upload_image_to_s3 +from zerver.models import Attachment, RealmEmoji, UserProfile s3backend = S3UploadBackend() diff --git a/zerver/lib/type_debug.py b/zerver/lib/type_debug.py index 35a07fbc45232..fed24fc977c67 100644 --- a/zerver/lib/type_debug.py +++ b/zerver/lib/type_debug.py @@ -1,7 +1,7 @@ -import sys import functools +import sys +from typing import IO, Any, Callable, Mapping, Sequence, TypeVar -from typing import Any, Callable, IO, Mapping, Sequence, TypeVar def get_mapping_type_str(x: Mapping[Any, Any]) -> str: container_type = type(x).__name__ diff --git a/zerver/lib/types.py b/zerver/lib/types.py index 04eb07320779c..ab274eeecdfb0 100644 --- a/zerver/lib/types.py +++ b/zerver/lib/types.py @@ -1,6 +1,7 @@ -from typing import TypeVar, Callable, Optional, List, Dict, Union, Tuple, Any -from typing_extensions import TypedDict +from typing import Any, Callable, Dict, List, Optional, Tuple, TypeVar, Union + from django.http import HttpResponse +from typing_extensions import TypedDict ViewFuncT = TypeVar('ViewFuncT', bound=Callable[..., HttpResponse]) diff --git a/zerver/lib/unminify.py b/zerver/lib/unminify.py index d6b41b720f695..3c19ea7a8d2d7 100644 --- a/zerver/lib/unminify.py +++ b/zerver/lib/unminify.py @@ -1,9 +1,9 @@ -import re import os -import sourcemap - +import re from typing import Dict, List +import sourcemap + from zerver.lib.pysa import mark_sanitized diff --git a/zerver/lib/upload.py b/zerver/lib/upload.py index 70f9db9ea6b84..65aa0278db0c2 100644 --- a/zerver/lib/upload.py +++ b/zerver/lib/upload.py @@ -1,45 +1,45 @@ -from typing import Any, Optional, Tuple - +import base64 +import binascii +import io +import logging +import os +import random +import re +import shutil +import sys +import unicodedata +import urllib from datetime import timedelta +from mimetypes import guess_extension, guess_type +from typing import Any, Optional, Tuple -from django.utils.translation import ugettext as _ +import boto3 +import botocore +from boto3.resources.base import ServiceResource +from boto3.session import Session +from botocore.client import Config from django.conf import settings from django.core.files import File -from django.core.signing import TimestampSigner, BadSignature +from django.core.signing import BadSignature, TimestampSigner from django.http import HttpRequest from django.urls import reverse +from django.utils.translation import ugettext as _ from jinja2 import Markup as mark_safe -import unicodedata +from PIL import ExifTags, Image, ImageOps +from PIL.GifImagePlugin import GifImageFile +from PIL.Image import DecompressionBombError from zerver.lib.avatar_hash import user_avatar_path -from zerver.lib.exceptions import JsonableError, ErrorCode +from zerver.lib.exceptions import ErrorCode, JsonableError from zerver.lib.utils import generate_random_token - -import boto3 -import botocore -from botocore.client import Config -from boto3.resources.base import ServiceResource -from boto3.session import Session - -from mimetypes import guess_type, guess_extension - -from zerver.models import get_user_profile_by_id -from zerver.models import Attachment -from zerver.models import Realm, RealmEmoji, UserProfile, Message - -import urllib -import base64 -import binascii -import os -import re -from PIL import Image, ImageOps, ExifTags -from PIL.Image import DecompressionBombError -from PIL.GifImagePlugin import GifImageFile -import io -import random -import logging -import shutil -import sys +from zerver.models import ( + Attachment, + Message, + Realm, + RealmEmoji, + UserProfile, + get_user_profile_by_id, +) DEFAULT_AVATAR_SIZE = 100 MEDIUM_AVATAR_SIZE = 500 diff --git a/zerver/lib/url_encoding.py b/zerver/lib/url_encoding.py index 3bd12dd276dd4..6c8058b90f543 100644 --- a/zerver/lib/url_encoding.py +++ b/zerver/lib/url_encoding.py @@ -5,6 +5,7 @@ from zerver.lib.topic import get_topic_from_message_info from zerver.models import Realm, Stream, UserProfile + def hash_util_encode(string: str) -> str: # Do the same encoding operation as hash_util.encodeHashComponent on the # frontend. diff --git a/zerver/lib/url_preview/oembed.py b/zerver/lib/url_preview/oembed.py index 16831f50a27de..b9451806b19a9 100644 --- a/zerver/lib/url_preview/oembed.py +++ b/zerver/lib/url_preview/oembed.py @@ -1,6 +1,8 @@ -from typing import Optional, Dict, Any -from pyoembed import oEmbed, PyOembedException import json +from typing import Any, Dict, Optional + +from pyoembed import PyOembedException, oEmbed + def get_oembed_data(url: str, maxwidth: Optional[int]=640, diff --git a/zerver/lib/url_preview/parsers/__init__.py b/zerver/lib/url_preview/parsers/__init__.py index 74168a2c3b53b..b812d115a0da0 100644 --- a/zerver/lib/url_preview/parsers/__init__.py +++ b/zerver/lib/url_preview/parsers/__init__.py @@ -1,4 +1,4 @@ -from zerver.lib.url_preview.parsers.open_graph import OpenGraphParser from zerver.lib.url_preview.parsers.generic import GenericParser +from zerver.lib.url_preview.parsers.open_graph import OpenGraphParser __all__ = ['OpenGraphParser', 'GenericParser'] diff --git a/zerver/lib/url_preview/parsers/base.py b/zerver/lib/url_preview/parsers/base.py index 25ed9e8ff6b1e..b173221302a70 100644 --- a/zerver/lib/url_preview/parsers/base.py +++ b/zerver/lib/url_preview/parsers/base.py @@ -1,5 +1,6 @@ from typing import Any + class BaseParser: def __init__(self, html_source: str) -> None: # We import BeautifulSoup here, because it's not used by most diff --git a/zerver/lib/url_preview/parsers/generic.py b/zerver/lib/url_preview/parsers/generic.py index a57db1664e584..d938114f36d30 100644 --- a/zerver/lib/url_preview/parsers/generic.py +++ b/zerver/lib/url_preview/parsers/generic.py @@ -1,4 +1,5 @@ from typing import Dict, Optional + from zerver.lib.url_preview.parsers.base import BaseParser diff --git a/zerver/lib/url_preview/parsers/open_graph.py b/zerver/lib/url_preview/parsers/open_graph.py index 75eee049e245f..8c31546e01603 100644 --- a/zerver/lib/url_preview/parsers/open_graph.py +++ b/zerver/lib/url_preview/parsers/open_graph.py @@ -1,4 +1,5 @@ from typing import Dict + from .base import BaseParser diff --git a/zerver/lib/url_preview/preview.py b/zerver/lib/url_preview/preview.py index f69d574043ee4..f764744100886 100644 --- a/zerver/lib/url_preview/preview.py +++ b/zerver/lib/url_preview/preview.py @@ -1,17 +1,17 @@ import re -import requests +from typing import Any, Callable, Dict, Optional +from typing.re import Match +import magic +import requests from django.conf import settings from django.utils.encoding import smart_text -import magic -from typing import Any, Optional, Dict, Callable -from typing.re import Match from version import ZULIP_VERSION from zerver.lib.cache import cache_with_key, get_cache_with_key, preview_url_cache_key from zerver.lib.pysa import mark_sanitized from zerver.lib.url_preview.oembed import get_oembed_data -from zerver.lib.url_preview.parsers import OpenGraphParser, GenericParser +from zerver.lib.url_preview.parsers import GenericParser, OpenGraphParser # FIXME: Should we use a database cache or a memcached in production? What if # opengraph data is changed for a site? diff --git a/zerver/lib/user_groups.py b/zerver/lib/user_groups.py index 9aec110b3c38e..5ce7209dc4331 100644 --- a/zerver/lib/user_groups.py +++ b/zerver/lib/user_groups.py @@ -1,8 +1,11 @@ +from typing import Any, Dict, List + from django.db import transaction from django.utils.translation import ugettext as _ + from zerver.lib.exceptions import JsonableError -from zerver.models import UserProfile, Realm, UserGroupMembership, UserGroup -from typing import Dict, List, Any +from zerver.models import Realm, UserGroup, UserGroupMembership, UserProfile + def access_user_group_by_id(user_group_id: int, user_profile: UserProfile) -> UserGroup: try: diff --git a/zerver/lib/user_status.py b/zerver/lib/user_status.py index fa2cca500b957..da4aa90f39f5b 100644 --- a/zerver/lib/user_status.py +++ b/zerver/lib/user_status.py @@ -1,11 +1,10 @@ +from typing import Any, Dict, Optional + from django.db.models import Q from django.utils.timezone import now as timezone_now -from zerver.models import ( - UserStatus, -) +from zerver.models import UserStatus -from typing import Any, Dict, Optional def get_user_info_dict(realm_id: int) -> Dict[int, Dict[str, Any]]: rows = UserStatus.objects.filter( diff --git a/zerver/lib/users.py b/zerver/lib/users.py index 61fe52318ef90..d92024540c9f2 100644 --- a/zerver/lib/users.py +++ b/zerver/lib/users.py @@ -1,24 +1,33 @@ -from typing import Any, Dict, List, Optional, Tuple, Union, cast - import re import unicodedata from collections import defaultdict +from typing import Any, Dict, List, Optional, Tuple, Union, cast from django.conf import settings from django.db.models.query import QuerySet from django.forms.models import model_to_dict from django.utils.translation import ugettext as _ +from zulip_bots.custom_exceptions import ConfigValidationError -from zerver.lib.cache import generic_bulk_cached_fetch, user_profile_cache_key_id, \ - user_profile_by_id_cache_key, realm_user_dict_fields -from zerver.lib.request import JsonableError from zerver.lib.avatar import avatar_url, get_avatar_field +from zerver.lib.cache import ( + generic_bulk_cached_fetch, + realm_user_dict_fields, + user_profile_by_id_cache_key, + user_profile_cache_key_id, +) from zerver.lib.exceptions import OrganizationAdministratorRequired -from zerver.models import UserProfile, Service, Realm, \ - get_user_profile_by_id_in_realm, CustomProfileFieldValue, \ - get_realm_user_dicts, CustomProfileField +from zerver.lib.request import JsonableError +from zerver.models import ( + CustomProfileField, + CustomProfileFieldValue, + Realm, + Service, + UserProfile, + get_realm_user_dicts, + get_user_profile_by_id_in_realm, +) -from zulip_bots.custom_exceptions import ConfigValidationError def check_full_name(full_name_raw: str) -> str: full_name = full_name_raw.strip() diff --git a/zerver/lib/utils.py b/zerver/lib/utils.py index 4b1d35ef8e568..59926fc8a89a6 100644 --- a/zerver/lib/utils.py +++ b/zerver/lib/utils.py @@ -1,4 +1,3 @@ -from typing import Any, Callable, List, Optional, Sequence, TypeVar, Iterable, Set, Tuple import base64 import hashlib import heapq @@ -6,8 +5,9 @@ import os import re import string -from time import sleep from itertools import zip_longest +from time import sleep +from typing import Any, Callable, Iterable, List, Optional, Sequence, Set, Tuple, TypeVar from django.conf import settings diff --git a/zerver/lib/validator.py b/zerver/lib/validator.py index 00920b0706965..59f4e24b6189b 100644 --- a/zerver/lib/validator.py +++ b/zerver/lib/validator.py @@ -26,17 +26,17 @@ for any particular type of object. ''' import re +from datetime import datetime +from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, TypeVar, Union, cast + import ujson -from django.utils.translation import ugettext as _ from django.conf import settings from django.core.exceptions import ValidationError -from django.core.validators import validate_email, URLValidator -from typing import Any, Dict, Iterable, Optional, Tuple, cast, List, Callable, TypeVar, \ - Set, Union +from django.core.validators import URLValidator, validate_email +from django.utils.translation import ugettext as _ -from datetime import datetime from zerver.lib.request import JsonableError -from zerver.lib.types import Validator, ProfileFieldData +from zerver.lib.types import ProfileFieldData, Validator FuncT = Callable[..., Any] TypeStructure = TypeVar("TypeStructure") diff --git a/zerver/lib/webhooks/common.py b/zerver/lib/webhooks/common.py index 442d5e406ff5a..3edd2ca0d2bb6 100644 --- a/zerver/lib/webhooks/common.py +++ b/zerver/lib/webhooks/common.py @@ -1,19 +1,25 @@ import importlib +from typing import Any, Callable, Dict, Optional, Union from urllib.parse import unquote from django.http import HttpRequest from django.utils.translation import ugettext as _ -from typing import Optional, Dict, Union, Any, Callable -from zerver.lib.actions import check_send_stream_message, \ - check_send_private_message, send_rate_limited_pm_notification_to_bot_owner -from zerver.lib.exceptions import StreamDoesNotExistError, JsonableError, \ - ErrorCode, UnexpectedWebhookEventType +from zerver.lib.actions import ( + check_send_private_message, + check_send_stream_message, + send_rate_limited_pm_notification_to_bot_owner, +) +from zerver.lib.exceptions import ( + ErrorCode, + JsonableError, + StreamDoesNotExistError, + UnexpectedWebhookEventType, +) from zerver.lib.request import REQ, has_request_variables from zerver.lib.send_email import FromAddress from zerver.models import UserProfile - MISSING_EVENT_HEADER_MESSAGE = """ Hi there! Your bot {bot_name} just sent an HTTP request to {request_path} that is missing the HTTP {header_name} header. Because this header is how diff --git a/zerver/lib/webhooks/git.py b/zerver/lib/webhooks/git.py index b57f33b843cfe..84b548e888fbd 100644 --- a/zerver/lib/webhooks/git.py +++ b/zerver/lib/webhooks/git.py @@ -1,6 +1,7 @@ import string -from typing import Optional, Any, Dict, List, Tuple from collections import defaultdict +from typing import Any, Dict, List, Optional, Tuple + TOPIC_WITH_BRANCH_TEMPLATE = '{repo} / {branch}' TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE = '{repo} / {type} #{id} {title}' TOPIC_WITH_RELEASE_TEMPLATE = '{repo} / {tag} {title}' diff --git a/zerver/lib/widget.py b/zerver/lib/widget.py index 319242c43bcde..fa3ad9cc3da76 100644 --- a/zerver/lib/widget.py +++ b/zerver/lib/widget.py @@ -1,7 +1,6 @@ -from typing import MutableMapping, Any, Optional, Tuple - -import re import json +import re +from typing import Any, MutableMapping, Optional, Tuple from zerver.models import SubMessage diff --git a/zerver/lib/zcommand.py b/zerver/lib/zcommand.py index d32253368ffd2..f6909b347d914 100644 --- a/zerver/lib/zcommand.py +++ b/zerver/lib/zcommand.py @@ -1,9 +1,11 @@ from typing import Any, Dict + from django.utils.translation import ugettext as _ -from zerver.models import UserProfile from zerver.lib.actions import do_set_user_display_setting from zerver.lib.exceptions import JsonableError +from zerver.models import UserProfile + def process_zcommands(content: str, user_profile: UserProfile) -> Dict[str, Any]: def change_mode_setting(command: str, switch_command: str, diff --git a/zerver/lib/zephyr.py b/zerver/lib/zephyr.py index c39b2f52a79e5..116ae1beae7f0 100644 --- a/zerver/lib/zephyr.py +++ b/zerver/lib/zephyr.py @@ -1,7 +1,9 @@ import re import traceback + import DNS + def compute_mit_user_fullname(email: str) -> str: try: # Input is either e.g. username@mit.edu or user|CROSSREALM.INVALID@mit.edu diff --git a/zerver/logging_handlers.py b/zerver/logging_handlers.py index 5b6539a0f00aa..19cb2bb18b4bc 100644 --- a/zerver/logging_handlers.py +++ b/zerver/logging_handlers.py @@ -1,8 +1,7 @@ # System documented in https://zulip.readthedocs.io/en/latest/subsystems/logging.html - import logging -import platform import os +import platform import subprocess import traceback from typing import Any, Dict, Optional @@ -11,9 +10,10 @@ from django.http import HttpRequest from django.views.debug import get_exception_reporter_filter +from version import ZULIP_VERSION from zerver.lib.logging_util import find_log_caller_module from zerver.lib.queue import queue_json_publish -from version import ZULIP_VERSION + def try_git_describe() -> Optional[str]: try: # nocoverage diff --git a/zerver/management/commands/check_redis.py b/zerver/management/commands/check_redis.py index d523de3f864d8..bc4cd91d1a891 100644 --- a/zerver/management/commands/check_redis.py +++ b/zerver/management/commands/check_redis.py @@ -3,8 +3,7 @@ from typing import Any, Callable, Optional from django.conf import settings -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.lib.rate_limiter import RateLimitedUser, client from zerver.models import get_user_profile_by_id diff --git a/zerver/management/commands/convert_gitter_data.py b/zerver/management/commands/convert_gitter_data.py index 62e88568c5dfe..e18bd893b91e1 100644 --- a/zerver/management/commands/convert_gitter_data.py +++ b/zerver/management/commands/convert_gitter_data.py @@ -3,8 +3,7 @@ import tempfile from typing import Any -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.data_import.gitter import do_convert_data diff --git a/zerver/management/commands/convert_hipchat_data.py b/zerver/management/commands/convert_hipchat_data.py index 9061c4476670c..e3f4e77cfe073 100644 --- a/zerver/management/commands/convert_hipchat_data.py +++ b/zerver/management/commands/convert_hipchat_data.py @@ -21,8 +21,7 @@ exporting-from-hipchat-server-or-data-center-for-data-portability-950821555.html ''' -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.data_import.hipchat import do_convert_data diff --git a/zerver/management/commands/convert_mattermost_data.py b/zerver/management/commands/convert_mattermost_data.py index d0e1df3bdd673..292a8b9c3d043 100644 --- a/zerver/management/commands/convert_mattermost_data.py +++ b/zerver/management/commands/convert_mattermost_data.py @@ -14,8 +14,7 @@ go to browser and use your dev url ''' -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.data_import.mattermost import do_convert_data diff --git a/zerver/management/commands/convert_slack_data.py b/zerver/management/commands/convert_slack_data.py index 7a75f0bbed033..5168146d14136 100644 --- a/zerver/management/commands/convert_slack_data.py +++ b/zerver/management/commands/convert_slack_data.py @@ -3,8 +3,7 @@ import tempfile from typing import Any -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.data_import.slack import do_convert_data diff --git a/zerver/management/commands/generate_invite_links.py b/zerver/management/commands/generate_invite_links.py index d6fdbe05fada6..cfb832469d98c 100644 --- a/zerver/management/commands/generate_invite_links.py +++ b/zerver/management/commands/generate_invite_links.py @@ -4,9 +4,9 @@ from django.core.management.base import CommandError from confirmation.models import Confirmation, create_confirmation_link +from zerver.lib.email_validation import email_allowed_for_realm from zerver.lib.management import ZulipBaseCommand from zerver.models import DomainNotAllowedForRealmError, PreregistrationUser -from zerver.lib.email_validation import email_allowed_for_realm class Command(ZulipBaseCommand): diff --git a/zerver/management/commands/import.py b/zerver/management/commands/import.py index 1f3406ecd8459..92282013c813a 100644 --- a/zerver/management/commands/import.py +++ b/zerver/management/commands/import.py @@ -5,8 +5,7 @@ from django.conf import settings from django.core.management import call_command -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.forms import check_subdomain_available from zerver.lib.import_realm import do_import_realm, do_import_system_bots diff --git a/zerver/management/commands/knight.py b/zerver/management/commands/knight.py index 51194ae72b6cf..61532c355a042 100644 --- a/zerver/management/commands/knight.py +++ b/zerver/management/commands/knight.py @@ -3,7 +3,7 @@ from django.core.management.base import CommandError -from zerver.lib.actions import do_change_user_role, do_change_is_api_super_user +from zerver.lib.actions import do_change_is_api_super_user, do_change_user_role from zerver.lib.management import ZulipBaseCommand from zerver.models import UserProfile diff --git a/zerver/management/commands/logout_all_users.py b/zerver/management/commands/logout_all_users.py index 6108643a580f0..b41b1c867c202 100644 --- a/zerver/management/commands/logout_all_users.py +++ b/zerver/management/commands/logout_all_users.py @@ -2,8 +2,11 @@ from typing import Any from zerver.lib.management import ZulipBaseCommand -from zerver.lib.sessions import delete_all_deactivated_user_sessions, \ - delete_all_user_sessions, delete_realm_user_sessions +from zerver.lib.sessions import ( + delete_all_deactivated_user_sessions, + delete_all_user_sessions, + delete_realm_user_sessions, +) class Command(ZulipBaseCommand): diff --git a/zerver/management/commands/makemessages.py b/zerver/management/commands/makemessages.py index 80f6220fd71f3..69ac464471e72 100644 --- a/zerver/management/commands/makemessages.py +++ b/zerver/management/commands/makemessages.py @@ -29,8 +29,8 @@ """ import glob -import json import itertools +import json import os import re from argparse import ArgumentParser diff --git a/zerver/management/commands/merge_streams.py b/zerver/management/commands/merge_streams.py index 517446da72f79..5d56f9e5a1771 100644 --- a/zerver/management/commands/merge_streams.py +++ b/zerver/management/commands/merge_streams.py @@ -1,8 +1,11 @@ from argparse import ArgumentParser from typing import Any, List -from zerver.lib.actions import bulk_add_subscriptions, \ - bulk_remove_subscriptions, do_deactivate_stream +from zerver.lib.actions import ( + bulk_add_subscriptions, + bulk_remove_subscriptions, + do_deactivate_stream, +) from zerver.lib.cache import cache_delete_many, to_dict_cache_key_id from zerver.lib.management import ZulipBaseCommand from zerver.models import Message, Subscription, get_stream diff --git a/zerver/management/commands/restore_messages.py b/zerver/management/commands/restore_messages.py index 9570658d01b3e..fa0b2a8b204c0 100644 --- a/zerver/management/commands/restore_messages.py +++ b/zerver/management/commands/restore_messages.py @@ -3,8 +3,11 @@ from django.core.management.base import CommandParser from zerver.lib.management import ZulipBaseCommand -from zerver.lib.retention import restore_all_data_from_archive, \ - restore_data_from_archive, restore_data_from_archive_by_realm +from zerver.lib.retention import ( + restore_all_data_from_archive, + restore_data_from_archive, + restore_data_from_archive_by_realm, +) from zerver.models import ArchiveTransaction diff --git a/zerver/management/commands/runtornado.py b/zerver/management/commands/runtornado.py index 02e27760c0ca6..de3adfe1ad726 100644 --- a/zerver/management/commands/runtornado.py +++ b/zerver/management/commands/runtornado.py @@ -3,8 +3,7 @@ from typing import Any, Callable from django.conf import settings -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from tornado import ioloop from tornado.log import app_log @@ -19,11 +18,14 @@ instrument_tornado_ioloop() from zerver.lib.debug import interactive_debug_listen -from zerver.tornado.application import create_tornado_application, \ - setup_tornado_rabbitmq +from zerver.tornado.application import create_tornado_application, setup_tornado_rabbitmq from zerver.tornado.autoreload import start as zulip_autoreload_start -from zerver.tornado.event_queue import add_client_gc_hook, \ - missedmessage_hook, get_wrapped_process_notification, setup_event_queue +from zerver.tornado.event_queue import ( + add_client_gc_hook, + get_wrapped_process_notification, + missedmessage_hook, + setup_event_queue, +) from zerver.tornado.sharding import notify_tornado_queue_name if settings.USING_RABBITMQ: diff --git a/zerver/management/commands/show_admins.py b/zerver/management/commands/show_admins.py index 7962adaee513e..f94479ec6a127 100644 --- a/zerver/management/commands/show_admins.py +++ b/zerver/management/commands/show_admins.py @@ -2,8 +2,10 @@ from typing import Any from django.core.management.base import CommandError + from zerver.lib.management import ZulipBaseCommand + class Command(ZulipBaseCommand): help = """Show the owners and administrators in an organization.""" diff --git a/zerver/management/commands/soft_deactivate_users.py b/zerver/management/commands/soft_deactivate_users.py index f379cd9edd18b..26481ab1ac050 100644 --- a/zerver/management/commands/soft_deactivate_users.py +++ b/zerver/management/commands/soft_deactivate_users.py @@ -6,8 +6,12 @@ from django.core.management.base import CommandError from zerver.lib.management import ZulipBaseCommand -from zerver.lib.soft_deactivation import do_auto_soft_deactivate_users, \ - do_soft_activate_users, do_soft_deactivate_users, logger +from zerver.lib.soft_deactivation import ( + do_auto_soft_deactivate_users, + do_soft_activate_users, + do_soft_deactivate_users, + logger, +) from zerver.models import Realm, UserProfile diff --git a/zerver/management/commands/transfer_uploads_to_s3.py b/zerver/management/commands/transfer_uploads_to_s3.py index d2c70f9f38078..77ade9cf48b2b 100644 --- a/zerver/management/commands/transfer_uploads_to_s3.py +++ b/zerver/management/commands/transfer_uploads_to_s3.py @@ -1,8 +1,7 @@ from typing import Any from django.conf import settings -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.lib.transfer import transfer_uploads_to_s3 diff --git a/zerver/middleware.py b/zerver/middleware.py index 3ceeb11576725..95795b64b3823 100644 --- a/zerver/middleware.py +++ b/zerver/middleware.py @@ -2,8 +2,7 @@ import logging import time import traceback -from typing import Any, AnyStr, Dict, \ - Iterable, List, MutableMapping, Optional +from typing import Any, AnyStr, Dict, Iterable, List, MutableMapping, Optional from django.conf import settings from django.core.exceptions import DisallowedHost @@ -17,15 +16,15 @@ from zerver.lib.bugdown import get_bugdown_requests, get_bugdown_time from zerver.lib.cache import get_remote_cache_requests, get_remote_cache_time -from zerver.lib.debug import maybe_tracemalloc_listen from zerver.lib.db import reset_queries +from zerver.lib.debug import maybe_tracemalloc_listen from zerver.lib.exceptions import ErrorCode, JsonableError, RateLimited from zerver.lib.html_to_text import get_content_description from zerver.lib.rate_limiter import RateLimitResult from zerver.lib.response import json_error, json_response_from_error from zerver.lib.subdomains import get_subdomain -from zerver.lib.utils import statsd from zerver.lib.types import ViewFuncT +from zerver.lib.utils import statsd from zerver.models import Realm, flush_per_request_caches, get_realm logger = logging.getLogger('zulip.requests') diff --git a/zerver/migrations/0030_realm_org_type.py b/zerver/migrations/0030_realm_org_type.py index 6f32a07e1ec9d..b086b48584aa6 100644 --- a/zerver/migrations/0030_realm_org_type.py +++ b/zerver/migrations/0030_realm_org_type.py @@ -1,6 +1,5 @@ from django.db import migrations, models - CORPORATE = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0032_verify_all_medium_avatar_images.py b/zerver/migrations/0032_verify_all_medium_avatar_images.py index f477ca52f78ee..ceb86eeb5a013 100644 --- a/zerver/migrations/0032_verify_all_medium_avatar_images.py +++ b/zerver/migrations/0032_verify_all_medium_avatar_images.py @@ -1,11 +1,11 @@ import hashlib from typing import Text +from unittest.mock import patch from django.conf import settings from django.db import migrations from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps -from unittest.mock import patch from zerver.lib.upload import upload_backend from zerver.lib.utils import make_safe_digest diff --git a/zerver/migrations/0038_realm_change_to_community_defaults.py b/zerver/migrations/0038_realm_change_to_community_defaults.py index ad8c885a0349f..50f65f00e5e43 100644 --- a/zerver/migrations/0038_realm_change_to_community_defaults.py +++ b/zerver/migrations/0038_realm_change_to_community_defaults.py @@ -1,6 +1,5 @@ from django.db import migrations, models - COMMUNITY = 2 class Migration(migrations.Migration): diff --git a/zerver/migrations/0060_move_avatars_to_be_uid_based.py b/zerver/migrations/0060_move_avatars_to_be_uid_based.py index 6200d286a7252..8d834eb38a225 100644 --- a/zerver/migrations/0060_move_avatars_to_be_uid_based.py +++ b/zerver/migrations/0060_move_avatars_to_be_uid_based.py @@ -1,6 +1,7 @@ # Generated by Django 1.10.5 on 2017-02-27 17:03 from django.db import migrations + class Migration(migrations.Migration): dependencies = [ diff --git a/zerver/migrations/0064_sync_uploads_filesize_with_db.py b/zerver/migrations/0064_sync_uploads_filesize_with_db.py index 674e98f23782a..337e1c6a81f80 100644 --- a/zerver/migrations/0064_sync_uploads_filesize_with_db.py +++ b/zerver/migrations/0064_sync_uploads_filesize_with_db.py @@ -1,6 +1,7 @@ # Generated by Django 1.10.5 on 2017-03-18 12:38 from django.db import migrations + class Migration(migrations.Migration): dependencies = [ diff --git a/zerver/migrations/0077_add_file_name_field_to_realm_emoji.py b/zerver/migrations/0077_add_file_name_field_to_realm_emoji.py index 62f513ec916c5..3d475d118cd4b 100644 --- a/zerver/migrations/0077_add_file_name_field_to_realm_emoji.py +++ b/zerver/migrations/0077_add_file_name_field_to_realm_emoji.py @@ -1,6 +1,7 @@ # Generated by Django 1.10.5 on 2017-03-09 05:23 from django.db import migrations, models + class Migration(migrations.Migration): dependencies = [ ('zerver', '0076_userprofile_emojiset'), diff --git a/zerver/migrations/0086_realm_alter_default_org_type.py b/zerver/migrations/0086_realm_alter_default_org_type.py index 1dca4623d85e6..6b5096d18097d 100644 --- a/zerver/migrations/0086_realm_alter_default_org_type.py +++ b/zerver/migrations/0086_realm_alter_default_org_type.py @@ -1,7 +1,6 @@ # Generated by Django 1.11.2 on 2017-06-26 21:56 from django.db import migrations, models - CORPORATE = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0143_realm_bot_creation_policy.py b/zerver/migrations/0143_realm_bot_creation_policy.py index 93414f9522932..7b8e7f0faa38a 100644 --- a/zerver/migrations/0143_realm_bot_creation_policy.py +++ b/zerver/migrations/0143_realm_bot_creation_policy.py @@ -1,10 +1,8 @@ # Generated by Django 1.11.6 on 2018-03-09 18:00 - from django.db import migrations, models from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps - BOT_CREATION_EVERYONE = 1 def set_initial_value_for_bot_creation_policy(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: Realm = apps.get_model("zerver", "Realm") diff --git a/zerver/migrations/0185_realm_plan_type.py b/zerver/migrations/0185_realm_plan_type.py index 3e9935ebcbf3a..c7349655df7d9 100644 --- a/zerver/migrations/0185_realm_plan_type.py +++ b/zerver/migrations/0185_realm_plan_type.py @@ -1,8 +1,6 @@ # Generated by Django 1.11.14 on 2018-08-10 21:36 - from django.db import migrations, models - SELF_HOSTED = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0214_realm_invite_to_stream_policy.py b/zerver/migrations/0214_realm_invite_to_stream_policy.py index 75e2fbbf56563..f9d3d85f2aad5 100644 --- a/zerver/migrations/0214_realm_invite_to_stream_policy.py +++ b/zerver/migrations/0214_realm_invite_to_stream_policy.py @@ -1,10 +1,8 @@ # Generated by Django 1.11.20 on 2019-04-29 05:29 - from django.db import migrations, models from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps - INVITE_TO_STREAM_POLICY_MEMBERS = 1 def handle_waiting_period(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: Realm = apps.get_model('zerver', 'Realm') diff --git a/zerver/migrations/0228_userprofile_demote_inactive_streams.py b/zerver/migrations/0228_userprofile_demote_inactive_streams.py index 30ac8d84fa409..bdb779d0b04c3 100644 --- a/zerver/migrations/0228_userprofile_demote_inactive_streams.py +++ b/zerver/migrations/0228_userprofile_demote_inactive_streams.py @@ -1,8 +1,6 @@ # Generated by Django 1.11.20 on 2019-03-08 19:50 - from django.db import migrations, models - DEMOTE_STREAMS_AUTOMATIC = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0235_userprofile_desktop_icon_count_display.py b/zerver/migrations/0235_userprofile_desktop_icon_count_display.py index 8ff03ae5b1242..c62476ff87414 100644 --- a/zerver/migrations/0235_userprofile_desktop_icon_count_display.py +++ b/zerver/migrations/0235_userprofile_desktop_icon_count_display.py @@ -1,8 +1,6 @@ # Generated by Django 1.11.20 on 2019-06-29 18:22 - from django.db import migrations, models - DESKTOP_ICON_COUNT_DISPLAY_MESSAGES = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0248_userprofile_role_start.py b/zerver/migrations/0248_userprofile_role_start.py index 03067678677f5..9408ef52dd179 100644 --- a/zerver/migrations/0248_userprofile_role_start.py +++ b/zerver/migrations/0248_userprofile_role_start.py @@ -1,9 +1,9 @@ # Generated by Django 1.11.24 on 2019-10-03 22:27 - from django.db import migrations, models from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps + def update_role(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: UserProfile = apps.get_model('zerver', 'UserProfile') # The values at the time of this migration diff --git a/zerver/migrations/0252_realm_user_group_edit_policy.py b/zerver/migrations/0252_realm_user_group_edit_policy.py index 25a1e61ee1b2f..193fc7b1dd485 100644 --- a/zerver/migrations/0252_realm_user_group_edit_policy.py +++ b/zerver/migrations/0252_realm_user_group_edit_policy.py @@ -1,8 +1,6 @@ # Generated by Django 1.11.24 on 2019-10-16 22:48 - from django.db import migrations, models - USER_GROUP_EDIT_POLICY_MEMBERS = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0261_realm_private_message_policy.py b/zerver/migrations/0261_realm_private_message_policy.py index 33a056a81e667..8688872bcc114 100644 --- a/zerver/migrations/0261_realm_private_message_policy.py +++ b/zerver/migrations/0261_realm_private_message_policy.py @@ -1,8 +1,6 @@ # Generated by Django 1.11.26 on 2020-01-08 00:32 - from django.db import migrations, models - PRIVATE_MESSAGE_POLICY_UNLIMITED = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0262_mutedtopic_date_muted.py b/zerver/migrations/0262_mutedtopic_date_muted.py index 22ee32a792d52..f42dd83e2b05e 100644 --- a/zerver/migrations/0262_mutedtopic_date_muted.py +++ b/zerver/migrations/0262_mutedtopic_date_muted.py @@ -1,6 +1,6 @@ # Generated by Django 1.11.26 on 2020-01-17 15:26 - import datetime + from django.db import migrations, models diff --git a/zerver/migrations/0263_stream_stream_post_policy.py b/zerver/migrations/0263_stream_stream_post_policy.py index 6cf2945ef9d9a..e619f5c859f8f 100644 --- a/zerver/migrations/0263_stream_stream_post_policy.py +++ b/zerver/migrations/0263_stream_stream_post_policy.py @@ -1,8 +1,6 @@ # Generated by Django 1.11.26 on 2020-01-27 22:03 - from django.db import migrations, models - STREAM_POST_POLICY_EVERYONE = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0266_userpresence_realm.py b/zerver/migrations/0266_userpresence_realm.py index 1937e5339c7cf..4d0d7cae9e930 100644 --- a/zerver/migrations/0266_userpresence_realm.py +++ b/zerver/migrations/0266_userpresence_realm.py @@ -1,7 +1,6 @@ # Generated by Django 1.11.28 on 2020-02-08 20:19 - -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/zerver/migrations/0268_add_userpresence_realm_timestamp_index.py b/zerver/migrations/0268_add_userpresence_realm_timestamp_index.py index 2eeb6fe187a42..9c28d383764a7 100644 --- a/zerver/migrations/0268_add_userpresence_realm_timestamp_index.py +++ b/zerver/migrations/0268_add_userpresence_realm_timestamp_index.py @@ -1,7 +1,6 @@ # Generated by Django 1.11.28 on 2020-02-08 20:34 - -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/zerver/migrations/0270_huddle_recipient.py b/zerver/migrations/0270_huddle_recipient.py index 9eac7771d7292..3273c63d74b95 100644 --- a/zerver/migrations/0270_huddle_recipient.py +++ b/zerver/migrations/0270_huddle_recipient.py @@ -1,7 +1,6 @@ # Generated by Django 2.2.10 on 2020-03-15 17:25 - -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/zerver/migrations/0273_migrate_old_bot_messages.py b/zerver/migrations/0273_migrate_old_bot_messages.py index 7cd7a0e8640ed..9494b093186b7 100644 --- a/zerver/migrations/0273_migrate_old_bot_messages.py +++ b/zerver/migrations/0273_migrate_old_bot_messages.py @@ -1,9 +1,10 @@ +from typing import Any + from django.conf import settings from django.db import migrations from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps -from typing import Any def fix_messages(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: """Conceptually, this migration cleans up the old NEW_USER_BOT and FEEDBACK_BOT diff --git a/zerver/migrations/0276_alertword.py b/zerver/migrations/0276_alertword.py index 75d3eafb3cb65..4de29acc54d0b 100644 --- a/zerver/migrations/0276_alertword.py +++ b/zerver/migrations/0276_alertword.py @@ -1,6 +1,6 @@ +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/zerver/migrations/0277_migrate_alert_word.py b/zerver/migrations/0277_migrate_alert_word.py index c4f755bde75d8..cca8edc8efbe3 100644 --- a/zerver/migrations/0277_migrate_alert_word.py +++ b/zerver/migrations/0277_migrate_alert_word.py @@ -1,8 +1,10 @@ +from typing import Dict, List + +import ujson from django.db import migrations from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps -import ujson -from typing import Dict, List + def move_to_seperate_table(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: UserProfile = apps.get_model('zerver', 'UserProfile') diff --git a/zerver/migrations/0284_convert_realm_admins_to_realm_owners.py b/zerver/migrations/0284_convert_realm_admins_to_realm_owners.py index 8e7463ffda773..bbd56d5e0e64b 100644 --- a/zerver/migrations/0284_convert_realm_admins_to_realm_owners.py +++ b/zerver/migrations/0284_convert_realm_admins_to_realm_owners.py @@ -1,12 +1,13 @@ # Generated by Django 2.2.12 on 2020-05-16 18:34 from typing import Any, Dict +import ujson from django.db import migrations from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps from django.db.models import Count from django.utils.timezone import now as timezone_now -import ujson + def set_realm_admins_as_realm_owners(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: UserProfile = apps.get_model('zerver', 'UserProfile') diff --git a/zerver/models.py b/zerver/models.py index d10c1d3ac4a9c..6e3fd7969cb75 100644 --- a/zerver/models.py +++ b/zerver/models.py @@ -1,52 +1,92 @@ -from typing import Any, DefaultDict, Dict, List, Set, Tuple, TypeVar, \ - Union, Optional, Sequence, AbstractSet, Callable, Iterable +import datetime +import re +import sre_constants +import time +from collections import defaultdict +from datetime import timedelta +from typing import ( + AbstractSet, + Any, + Callable, + DefaultDict, + Dict, + Iterable, + List, + Optional, + Sequence, + Set, + Tuple, + TypeVar, + Union, +) -from django.db import models -from django.db.models.query import QuerySet -from django.db.models import Manager, Q, Sum, CASCADE -from django.conf import settings -from django.contrib.auth.models import AbstractBaseUser, UserManager, \ - PermissionsMixin import django.contrib.auth +from bitfield import BitField +from bitfield.types import BitHandler +from django.conf import settings +from django.contrib.auth.models import AbstractBaseUser, PermissionsMixin, UserManager +from django.contrib.postgres.fields import JSONField from django.core.exceptions import ValidationError -from django.core.validators import URLValidator, MinLengthValidator, \ - RegexValidator, validate_email -from zerver.lib.cache import cache_with_key, flush_user_profile, flush_realm, \ - user_profile_by_api_key_cache_key, active_non_guest_user_ids_cache_key, \ - user_profile_by_id_cache_key, user_profile_by_email_cache_key, \ - user_profile_cache_key, generic_bulk_cached_fetch, cache_set, flush_stream, \ - cache_delete, active_user_ids_cache_key, \ - get_stream_cache_key, realm_user_dicts_cache_key, \ - bot_dicts_in_realm_cache_key, realm_user_dict_fields, \ - bot_dict_fields, flush_message, flush_submessage, bot_profile_cache_key, \ - flush_used_upload_space_cache, get_realm_used_upload_space_cache_key, \ - realm_alert_words_cache_key, realm_alert_words_automaton_cache_key -from zerver.lib.utils import make_safe_digest, generate_random_token -from django.db import transaction +from django.core.validators import MinLengthValidator, RegexValidator, URLValidator, validate_email +from django.db import models, transaction +from django.db.models import CASCADE, Manager, Q, Sum +from django.db.models.query import QuerySet +from django.db.models.signals import post_delete, post_save from django.utils.timezone import now as timezone_now -from zerver.lib.timestamp import datetime_to_timestamp -from django.db.models.signals import post_save, post_delete from django.utils.translation import ugettext_lazy as _ + from zerver.lib import cache -from zerver.lib.pysa import mark_sanitized -from zerver.lib.validator import check_int, \ - check_short_string, check_long_string, validate_choice_field, check_date, \ - check_url, check_list -from zerver.lib.types import Validator, ExtendedValidator, \ - ProfileDataElement, ProfileData, RealmUserValidator, \ - ExtendedFieldElement, UserFieldElement, FieldElement, \ - DisplayRecipientT +from zerver.lib.cache import ( + active_non_guest_user_ids_cache_key, + active_user_ids_cache_key, + bot_dict_fields, + bot_dicts_in_realm_cache_key, + bot_profile_cache_key, + cache_delete, + cache_set, + cache_with_key, + flush_message, + flush_realm, + flush_stream, + flush_submessage, + flush_used_upload_space_cache, + flush_user_profile, + generic_bulk_cached_fetch, + get_realm_used_upload_space_cache_key, + get_stream_cache_key, + realm_alert_words_automaton_cache_key, + realm_alert_words_cache_key, + realm_user_dict_fields, + realm_user_dicts_cache_key, + user_profile_by_api_key_cache_key, + user_profile_by_email_cache_key, + user_profile_by_id_cache_key, + user_profile_cache_key, +) from zerver.lib.exceptions import JsonableError -from django.contrib.postgres.fields import JSONField - -from bitfield import BitField -from bitfield.types import BitHandler -from collections import defaultdict -from datetime import timedelta -import re -import sre_constants -import time -import datetime +from zerver.lib.pysa import mark_sanitized +from zerver.lib.timestamp import datetime_to_timestamp +from zerver.lib.types import ( + DisplayRecipientT, + ExtendedFieldElement, + ExtendedValidator, + FieldElement, + ProfileData, + ProfileDataElement, + RealmUserValidator, + UserFieldElement, + Validator, +) +from zerver.lib.utils import generate_random_token, make_safe_digest +from zerver.lib.validator import ( + check_date, + check_int, + check_list, + check_long_string, + check_short_string, + check_url, + validate_choice_field, +) MAX_TOPIC_NAME_LENGTH = 60 MAX_MESSAGE_LENGTH = 10000 diff --git a/zerver/openapi/curl_param_value_generators.py b/zerver/openapi/curl_param_value_generators.py index 59cb4d68e042b..b6ebadd67a224 100644 --- a/zerver/openapi/curl_param_value_generators.py +++ b/zerver/openapi/curl_param_value_generators.py @@ -1,13 +1,12 @@ -from typing import Dict, Any, Callable, Set, List, Optional, Tuple - from functools import wraps +from typing import Any, Callable, Dict, List, Optional, Set, Tuple from django.utils.timezone import now as timezone_now -from zerver.models import Client, Message, UserPresence, UserGroup, get_realm -from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.actions import do_add_reaction, do_add_realm_filter, update_user_presence from zerver.lib.events import do_events_register -from zerver.lib.actions import update_user_presence, do_add_realm_filter, do_add_reaction +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import Client, Message, UserGroup, UserPresence, get_realm GENERATOR_FUNCTIONS: Dict[str, Callable[..., Dict[Any, Any]]] = dict() REGISTERED_GENERATOR_FUNCTIONS: Set[str] = set() diff --git a/zerver/openapi/javascript_examples.py b/zerver/openapi/javascript_examples.py index cbb41a1457343..47318cda2bac9 100644 --- a/zerver/openapi/javascript_examples.py +++ b/zerver/openapi/javascript_examples.py @@ -1,10 +1,12 @@ -import os import json +import os import subprocess from zulip import Client + from zerver.openapi.openapi import validate_against_openapi_schema + def test_js_bindings(client: Client) -> None: os.environ['ZULIP_USERNAME'] = client.email os.environ['ZULIP_API_KEY'] = client.api_key diff --git a/zerver/openapi/markdown_extension.py b/zerver/openapi/markdown_extension.py index 1a6c5d946b2d0..4dc85f88b71bf 100644 --- a/zerver/openapi/markdown_extension.py +++ b/zerver/openapi/markdown_extension.py @@ -1,16 +1,15 @@ -import re -import json import inspect +import json +import re +from typing import Any, Dict, List, Optional, Pattern, Tuple +import markdown from django.conf import settings - from markdown.extensions import Extension from markdown.preprocessors import Preprocessor -from typing import Any, Dict, Optional, List, Tuple, Pattern -import markdown import zerver.openapi.python_examples -from zerver.openapi.openapi import get_openapi_fixture, openapi_spec, get_openapi_description +from zerver.openapi.openapi import get_openapi_description, get_openapi_fixture, openapi_spec MACRO_REGEXP = re.compile( r'\{generate_code_example(\(\s*(.+?)\s*\))*\|\s*(.+?)\s*\|\s*(.+?)\s*(\(\s*(.+)\s*\))?\}') diff --git a/zerver/openapi/python_examples.py b/zerver/openapi/python_examples.py index abcd4e864b28b..885ff60594129 100644 --- a/zerver/openapi/python_examples.py +++ b/zerver/openapi/python_examples.py @@ -1,16 +1,14 @@ -from typing import Dict, Any, Optional, Iterable, Callable, Set, List - import json import os import sys from functools import wraps +from typing import Any, Callable, Dict, Iterable, List, Optional, Set -from zerver.lib import mdiff -from zerver.openapi.openapi import validate_against_openapi_schema +from zulip import Client +from zerver.lib import mdiff from zerver.models import get_realm, get_user - -from zulip import Client +from zerver.openapi.openapi import validate_against_openapi_schema ZULIP_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/zerver/openapi/test_curl_examples.py b/zerver/openapi/test_curl_examples.py index 9d1ec803cea4a..f2e6a268c3ea0 100644 --- a/zerver/openapi/test_curl_examples.py +++ b/zerver/openapi/test_curl_examples.py @@ -1,14 +1,19 @@ import glob +import html import json import shlex import subprocess -import markdown -import html +import markdown from zulip import Client -from zerver.openapi import markdown_extension + from zerver.models import get_realm -from zerver.openapi.curl_param_value_generators import REGISTERED_GENERATOR_FUNCTIONS, CALLED_GENERATOR_FUNCTIONS +from zerver.openapi import markdown_extension +from zerver.openapi.curl_param_value_generators import ( + CALLED_GENERATOR_FUNCTIONS, + REGISTERED_GENERATOR_FUNCTIONS, +) + def test_generated_curl_examples_for_success(client: Client) -> None: authentication_line = f"{client.email}:{client.api_key}" diff --git a/zerver/signals.py b/zerver/signals.py index ddcf2c080b783..86a40dfe143d8 100644 --- a/zerver/signals.py +++ b/zerver/signals.py @@ -3,8 +3,7 @@ from django.conf import settings from django.contrib.auth.signals import user_logged_in, user_logged_out from django.dispatch import receiver -from django.utils.timezone import \ - get_current_timezone_name as timezone_get_current_timezone_name +from django.utils.timezone import get_current_timezone_name as timezone_get_current_timezone_name from django.utils.timezone import now as timezone_now from django.utils.translation import ugettext as _ @@ -12,8 +11,8 @@ from zerver.lib.actions import do_set_zoom_token from zerver.lib.queue import queue_json_publish from zerver.lib.send_email import FromAddress -from zerver.models import UserProfile from zerver.lib.timezone import get_timezone +from zerver.models import UserProfile JUST_CREATED_THRESHOLD = 60 diff --git a/zerver/templatetags/app_filters.py b/zerver/templatetags/app_filters.py index ad5437159c053..2c23d10f17583 100644 --- a/zerver/templatetags/app_filters.py +++ b/zerver/templatetags/app_filters.py @@ -9,17 +9,17 @@ from django.utils.safestring import mark_safe from jinja2.exceptions import TemplateNotFound -import zerver.lib.bugdown.fenced_code import zerver.lib.bugdown.api_arguments_table_generator import zerver.lib.bugdown.api_return_values_table_generator -import zerver.openapi.markdown_extension -import zerver.lib.bugdown.nested_code_blocks -import zerver.lib.bugdown.tabbed_sections -import zerver.lib.bugdown.help_settings_links -import zerver.lib.bugdown.help_relative_links +import zerver.lib.bugdown.fenced_code import zerver.lib.bugdown.help_emoticon_translations_table +import zerver.lib.bugdown.help_relative_links +import zerver.lib.bugdown.help_settings_links import zerver.lib.bugdown.include -from zerver.lib.cache import ignore_unhashable_lru_cache, dict_to_items_tuple, items_tuple_to_dict +import zerver.lib.bugdown.nested_code_blocks +import zerver.lib.bugdown.tabbed_sections +import zerver.openapi.markdown_extension +from zerver.lib.cache import dict_to_items_tuple, ignore_unhashable_lru_cache, items_tuple_to_dict register = Library() diff --git a/zerver/tests/test_alert_words.py b/zerver/tests/test_alert_words.py index 1ca4d37534bb2..4dd5583f364f5 100644 --- a/zerver/tests/test_alert_words.py +++ b/zerver/tests/test_alert_words.py @@ -1,27 +1,11 @@ -from zerver.lib.alert_words import ( - alert_words_in_realm, - user_alert_words, -) - -from zerver.lib.actions import ( - do_add_alert_words, - do_remove_alert_words, -) - -from zerver.lib.test_helpers import ( - most_recent_message, - most_recent_usermessage, -) - -from zerver.lib.test_classes import ( - ZulipTestCase, -) +import ujson -from zerver.models import ( - UserProfile, -) +from zerver.lib.actions import do_add_alert_words, do_remove_alert_words +from zerver.lib.alert_words import alert_words_in_realm, user_alert_words +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import most_recent_message, most_recent_usermessage +from zerver.models import UserProfile -import ujson class AlertWordTests(ZulipTestCase): interesting_alert_word_list = ['alert', 'multi-word word', '☃'] diff --git a/zerver/tests/test_archive.py b/zerver/tests/test_archive.py index c42ecbe322204..24a333ac94b5d 100644 --- a/zerver/tests/test_archive.py +++ b/zerver/tests/test_archive.py @@ -1,10 +1,15 @@ from django.http import HttpResponse + +from zerver.lib.actions import ( + do_change_stream_web_public, + do_deactivate_stream, + get_web_public_streams, + get_web_public_subs, +) from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.actions import do_change_stream_web_public -from zerver.lib.actions import get_web_public_streams, get_web_public_subs, \ - do_deactivate_stream from zerver.models import get_realm + class GlobalPublicStreamTest(ZulipTestCase): def test_non_existant_stream_id(self) -> None: # Here we use a relatively big number as stream id assuming such an id diff --git a/zerver/tests/test_attachments.py b/zerver/tests/test_attachments.py index 5fefdb9471939..7b8ec3b832a8b 100644 --- a/zerver/tests/test_attachments.py +++ b/zerver/tests/test_attachments.py @@ -1,6 +1,5 @@ -from unittest import mock - from typing import Any +from unittest import mock from zerver.lib.attachments import user_attachments from zerver.lib.test_classes import ZulipTestCase diff --git a/zerver/tests/test_audit_log.py b/zerver/tests/test_audit_log.py index 34ed7440673d1..8718eda40a733 100644 --- a/zerver/tests/test_audit_log.py +++ b/zerver/tests/test_audit_log.py @@ -1,20 +1,32 @@ +from datetime import timedelta +from typing import Any, Dict + +import ujson +from django.contrib.auth.password_validation import validate_password from django.utils.timezone import now as timezone_now -from zerver.lib.actions import do_create_user, do_deactivate_user, \ - do_activate_user, do_reactivate_user, do_change_password, \ - do_change_user_delivery_email, do_change_avatar_fields, do_change_bot_owner, \ - do_regenerate_api_key, do_change_tos_version, \ - bulk_add_subscriptions, bulk_remove_subscriptions, get_streams_traffic, \ - do_change_user_role, do_deactivate_realm, do_reactivate_realm -from zerver.lib.test_classes import ZulipTestCase -from zerver.models import RealmAuditLog, get_client, get_realm, UserProfile from analytics.models import StreamCount +from zerver.lib.actions import ( + bulk_add_subscriptions, + bulk_remove_subscriptions, + do_activate_user, + do_change_avatar_fields, + do_change_bot_owner, + do_change_password, + do_change_tos_version, + do_change_user_delivery_email, + do_change_user_role, + do_create_user, + do_deactivate_realm, + do_deactivate_user, + do_reactivate_realm, + do_reactivate_user, + do_regenerate_api_key, + get_streams_traffic, +) +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import RealmAuditLog, UserProfile, get_client, get_realm -from datetime import timedelta -from django.contrib.auth.password_validation import validate_password - -from typing import Any, Dict -import ujson class TestRealmAuditLog(ZulipTestCase): def check_role_count_schema(self, role_counts: Dict[str, Any]) -> None: diff --git a/zerver/tests/test_auth_backends.py b/zerver/tests/test_auth_backends.py index 992cbc27e772c..6621cba44b836 100644 --- a/zerver/tests/test_auth_backends.py +++ b/zerver/tests/test_auth_backends.py @@ -1,29 +1,39 @@ +import base64 +import copy +import datetime +import json +import re +import time +import urllib +from typing import Any, Callable, Dict, List, Optional, Tuple +from unittest import mock + +import jwt +import ldap +import requests +import responses +import ujson from bs4 import BeautifulSoup from cryptography.hazmat.primitives.ciphers.aead import AESGCM from django.conf import settings from django.contrib.auth import authenticate from django.core import mail -from django.http import HttpResponse, HttpRequest +from django.http import HttpRequest, HttpResponse from django.test import override_settings -from django_auth_ldap.backend import LDAPSearch, _LDAPUser from django.test.client import RequestFactory -from django.utils.timezone import now as timezone_now -from typing import Any, Callable, Dict, List, Optional, Tuple from django.urls import reverse +from django.utils.timezone import now as timezone_now +from django_auth_ldap.backend import LDAPSearch, _LDAPUser +from onelogin.saml2.auth import OneLogin_Saml2_Auth +from onelogin.saml2.response import OneLogin_Saml2_Response +from social_core.exceptions import AuthFailed, AuthStateForbidden +from social_django.storage import BaseDjangoStorage +from social_django.strategy import DjangoStrategy -import responses - -import ldap -import jwt -from unittest import mock -import re -import datetime -import time -import requests - +from confirmation.models import Confirmation, create_confirmation_link from zerver.lib.actions import ( - do_create_user, do_create_realm, + do_create_user, do_deactivate_realm, do_deactivate_user, do_invite_users, @@ -35,57 +45,92 @@ from zerver.lib.avatar import avatar_url from zerver.lib.avatar_hash import user_avatar_path from zerver.lib.dev_ldap_directory import generate_dev_ldap_dir -from zerver.lib.email_validation import get_realm_email_validator, \ - validate_email_is_valid, get_existing_user_errors +from zerver.lib.email_validation import ( + get_existing_user_errors, + get_realm_email_validator, + validate_email_is_valid, +) from zerver.lib.exceptions import RateLimited +from zerver.lib.initial_password import initial_password from zerver.lib.mobile_auth_otp import otp_decrypt_api_key -from zerver.lib.validator import validate_login_email, \ - check_bool, check_dict_only, check_list, check_string, check_int, Validator from zerver.lib.rate_limiter import add_ratelimit_rule, remove_ratelimit_rule from zerver.lib.request import JsonableError from zerver.lib.storage import static_path -from zerver.lib.upload import resize_avatar, MEDIUM_AVATAR_SIZE +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import ( + create_s3_buckets, + get_test_image_file, + load_subdomain_token, + use_s3_backend, +) +from zerver.lib.upload import MEDIUM_AVATAR_SIZE, resize_avatar from zerver.lib.users import get_all_api_keys from zerver.lib.utils import generate_random_token -from zerver.lib.initial_password import initial_password -from zerver.lib.test_classes import ( - ZulipTestCase, +from zerver.lib.validator import ( + Validator, + check_bool, + check_dict_only, + check_int, + check_list, + check_string, + validate_login_email, +) +from zerver.models import ( + CustomProfileField, + CustomProfileFieldValue, + MultiuseInvite, + PasswordTooWeakError, + PreregistrationUser, + Realm, + RealmDomain, + UserProfile, + clear_supported_auth_backends_cache, + email_to_username, + get_realm, + get_user_by_delivery_email, ) -from zerver.models import \ - get_realm, email_to_username, CustomProfileField, CustomProfileFieldValue, \ - UserProfile, PreregistrationUser, Realm, RealmDomain, MultiuseInvite, \ - clear_supported_auth_backends_cache, PasswordTooWeakError, get_user_by_delivery_email from zerver.signals import JUST_CREATED_THRESHOLD - -from confirmation.models import Confirmation, create_confirmation_link - -from zproject.backends import ZulipDummyBackend, EmailAuthBackend, AppleAuthBackend, \ - GoogleAuthBackend, ZulipRemoteUserBackend, ZulipLDAPAuthBackend, \ - ZulipLDAPUserPopulator, DevAuthBackend, GitHubAuthBackend, GitLabAuthBackend, ZulipAuthMixin, \ - dev_auth_enabled, password_auth_enabled, github_auth_enabled, gitlab_auth_enabled, \ - apple_auth_enabled, google_auth_enabled, require_email_format_usernames, AUTH_BACKEND_NAME_MAP, \ - ZulipLDAPConfigurationError, ZulipLDAPExceptionNoMatchingLDAPUser, ZulipLDAPExceptionOutsideDomain, \ - ZulipLDAPException, query_ldap, sync_user_from_ldap, SocialAuthMixin, \ - PopulateUserLDAPError, SAMLAuthBackend, saml_auth_enabled, email_belongs_to_ldap, \ - get_external_method_dicts, AzureADAuthBackend, check_password_strength, \ - ZulipLDAPUser, RateLimitedAuthenticationByUsername, ExternalAuthResult, \ - ExternalAuthDataDict - from zerver.views.auth import maybe_send_to_registration +from zproject.backends import ( + AUTH_BACKEND_NAME_MAP, + AppleAuthBackend, + AzureADAuthBackend, + DevAuthBackend, + EmailAuthBackend, + ExternalAuthDataDict, + ExternalAuthResult, + GitHubAuthBackend, + GitLabAuthBackend, + GoogleAuthBackend, + PopulateUserLDAPError, + RateLimitedAuthenticationByUsername, + SAMLAuthBackend, + SocialAuthMixin, + ZulipAuthMixin, + ZulipDummyBackend, + ZulipLDAPAuthBackend, + ZulipLDAPConfigurationError, + ZulipLDAPException, + ZulipLDAPExceptionNoMatchingLDAPUser, + ZulipLDAPExceptionOutsideDomain, + ZulipLDAPUser, + ZulipLDAPUserPopulator, + ZulipRemoteUserBackend, + apple_auth_enabled, + check_password_strength, + dev_auth_enabled, + email_belongs_to_ldap, + get_external_method_dicts, + github_auth_enabled, + gitlab_auth_enabled, + google_auth_enabled, + password_auth_enabled, + query_ldap, + require_email_format_usernames, + saml_auth_enabled, + sync_user_from_ldap, +) -from onelogin.saml2.auth import OneLogin_Saml2_Auth -from onelogin.saml2.response import OneLogin_Saml2_Response -from social_core.exceptions import AuthFailed, AuthStateForbidden -from social_django.strategy import DjangoStrategy -from social_django.storage import BaseDjangoStorage - -import base64 -import copy -import json -import urllib -import ujson -from zerver.lib.test_helpers import load_subdomain_token, \ - use_s3_backend, create_s3_buckets, get_test_image_file class AuthBackendTest(ZulipTestCase): def get_username(self, email_to_username: Optional[Callable[[str], str]]=None) -> str: diff --git a/zerver/tests/test_bots.py b/zerver/tests/test_bots.py index a1e1069456335..40267ecee79a9 100644 --- a/zerver/tests/test_bots.py +++ b/zerver/tests/test_bots.py @@ -1,28 +1,38 @@ import filecmp import os -import ujson +from typing import Any, Dict, List, Mapping, Optional +from unittest.mock import MagicMock, patch +import ujson from django.core import mail from django.test import override_settings -from unittest.mock import patch, MagicMock -from typing import Any, Dict, List, Mapping, Optional +from zulip_bots.custom_exceptions import ConfigValidationError -from zerver.lib.actions import do_change_stream_invite_only, do_deactivate_user, \ - do_set_realm_property -from zerver.lib.bot_config import get_bot_config, ConfigError -from zerver.models import get_realm, get_stream, \ - Realm, UserProfile, get_user, get_bot_services, Service, \ - is_cross_realm_bot_email -from zerver.lib.test_classes import ZulipTestCase, UploadSerializeMixin +from zerver.lib.actions import ( + do_change_stream_invite_only, + do_deactivate_user, + do_set_realm_property, +) +from zerver.lib.bot_config import ConfigError, get_bot_config +from zerver.lib.bot_lib import get_bot_handler +from zerver.lib.integrations import EMBEDDED_BOTS, WebhookIntegration +from zerver.lib.test_classes import UploadSerializeMixin, ZulipTestCase from zerver.lib.test_helpers import ( avatar_disk_path, get_test_image_file, queries_captured, tornado_redirected_to_list, ) -from zerver.lib.integrations import EMBEDDED_BOTS, WebhookIntegration -from zerver.lib.bot_lib import get_bot_handler -from zulip_bots.custom_exceptions import ConfigValidationError +from zerver.models import ( + Realm, + Service, + UserProfile, + get_bot_services, + get_realm, + get_stream, + get_user, + is_cross_realm_bot_email, +) # A test validator diff --git a/zerver/tests/test_bugdown.py b/zerver/tests/test_bugdown.py index f3e7efb2b9290..b8311d825984b 100644 --- a/zerver/tests/test_bugdown.py +++ b/zerver/tests/test_bugdown.py @@ -1,12 +1,19 @@ +import copy +import os +import re +from typing import Any, Dict, List, Optional, Set, Tuple, cast +from unittest import mock + +import ujson from django.conf import settings from django.test import TestCase, override_settings -from zerver.lib import bugdown +from zerver.lib import bugdown, mdiff from zerver.lib.actions import ( - do_set_user_display_setting, - do_remove_realm_emoji, do_add_alert_words, + do_remove_realm_emoji, do_set_realm_property, + do_set_user_display_setting, ) from zerver.lib.alert_words import get_alert_word_automaton from zerver.lib.create_user import create_user @@ -14,42 +21,30 @@ from zerver.lib.exceptions import BugdownRenderingException from zerver.lib.mention import possible_mentions, possible_user_group_mentions from zerver.lib.message import render_markdown -from zerver.lib.request import ( - JsonableError, -) -from zerver.lib.user_groups import create_user_group -from zerver.lib.test_classes import ( - ZulipTestCase, -) +from zerver.lib.request import JsonableError +from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_runner import slow -from zerver.lib import mdiff from zerver.lib.tex import render_tex +from zerver.lib.user_groups import create_user_group from zerver.models import ( - realm_in_local_realm_filters_cache, - flush_per_request_caches, - flush_realm_filter, - get_client, - get_realm, - get_stream, - realm_filters_for_realm, MAX_MESSAGE_LENGTH, Message, - Stream, Realm, RealmEmoji, RealmFilter, + Stream, + UserGroup, UserMessage, UserProfile, - UserGroup, + flush_per_request_caches, + flush_realm_filter, + get_client, + get_realm, + get_stream, + realm_filters_for_realm, + realm_in_local_realm_filters_cache, ) -import copy -from unittest import mock -import os -import ujson -import re - -from typing import cast, Any, Dict, List, Optional, Set, Tuple class FakeMessage: pass diff --git a/zerver/tests/test_cache.py b/zerver/tests/test_cache.py index d9c1a32482563..a28455a856a16 100644 --- a/zerver/tests/test_cache.py +++ b/zerver/tests/test_cache.py @@ -1,16 +1,31 @@ -from django.conf import settings - +from typing import Any, Dict, List, Optional from unittest.mock import Mock, patch -from typing import Any, List, Dict, Optional + +from django.conf import settings from zerver.apps import flush_cache -from zerver.lib.cache import generic_bulk_cached_fetch, user_profile_by_email_cache_key, cache_with_key, \ - validate_cache_key, InvalidCacheKeyException, MEMCACHED_MAX_KEY_LENGTH, get_cache_with_key, \ - NotFoundInCache, cache_set, cache_get, cache_delete, cache_delete_many, cache_get_many, cache_set_many, \ - safe_cache_get_many, safe_cache_set_many +from zerver.lib.cache import ( + MEMCACHED_MAX_KEY_LENGTH, + InvalidCacheKeyException, + NotFoundInCache, + cache_delete, + cache_delete_many, + cache_get, + cache_get_many, + cache_set, + cache_set_many, + cache_with_key, + generic_bulk_cached_fetch, + get_cache_with_key, + safe_cache_get_many, + safe_cache_set_many, + user_profile_by_email_cache_key, + validate_cache_key, +) from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import queries_captured -from zerver.models import get_system_bot, get_user_profile_by_email, UserProfile +from zerver.models import UserProfile, get_system_bot, get_user_profile_by_email + class AppsTest(ZulipTestCase): def test_cache_gets_flushed(self) -> None: diff --git a/zerver/tests/test_camo.py b/zerver/tests/test_camo.py index 8224047a3106f..3ef7c1e28af07 100644 --- a/zerver/tests/test_camo.py +++ b/zerver/tests/test_camo.py @@ -1,5 +1,6 @@ from zerver.lib.test_classes import ZulipTestCase + class CamoURLTest(ZulipTestCase): def test_legacy_camo_url(self) -> None: # Test with valid hex and url pair diff --git a/zerver/tests/test_compatibility.py b/zerver/tests/test_compatibility.py index 6f21f510d2f96..0b8101d3083ef 100644 --- a/zerver/tests/test_compatibility.py +++ b/zerver/tests/test_compatibility.py @@ -1,7 +1,7 @@ from unittest import mock from zerver.lib.test_classes import ZulipTestCase -from zerver.views.compatibility import find_mobile_os, version_lt, is_outdated_desktop_app +from zerver.views.compatibility import find_mobile_os, is_outdated_desktop_app, version_lt class VersionTest(ZulipTestCase): diff --git a/zerver/tests/test_create_video_call.py b/zerver/tests/test_create_video_call.py index f4c1bbb238066..e393b1da2d402 100644 --- a/zerver/tests/test_create_video_call.py +++ b/zerver/tests/test_create_video_call.py @@ -1,4 +1,5 @@ import responses + from zerver.lib.test_classes import ZulipTestCase diff --git a/zerver/tests/test_custom_profile_data.py b/zerver/tests/test_custom_profile_data.py index 6f2cf7d6b8292..7889531f4df83 100644 --- a/zerver/tests/test_custom_profile_data.py +++ b/zerver/tests/test_custom_profile_data.py @@ -1,17 +1,25 @@ -from typing import Union, List, Dict, Any +from typing import Any, Dict, List, Union +from unittest import mock -from zerver.lib.actions import try_add_realm_custom_profile_field, \ - do_update_user_custom_profile_data_if_changed, do_remove_realm_custom_profile_field, \ - try_reorder_realm_custom_profile_fields -from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.test_helpers import queries_captured +import ujson + +from zerver.lib.actions import ( + do_remove_realm_custom_profile_field, + do_update_user_custom_profile_data_if_changed, + try_add_realm_custom_profile_field, + try_reorder_realm_custom_profile_fields, +) from zerver.lib.bugdown import convert as bugdown_convert -from zerver.models import CustomProfileField, \ - custom_profile_fields_for_realm, CustomProfileFieldValue, get_realm from zerver.lib.external_accounts import DEFAULT_EXTERNAL_ACCOUNTS -import ujson +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import queries_captured +from zerver.models import ( + CustomProfileField, + CustomProfileFieldValue, + custom_profile_fields_for_realm, + get_realm, +) -from unittest import mock class CustomProfileFieldTestCase(ZulipTestCase): def setUp(self) -> None: diff --git a/zerver/tests/test_decorators.py b/zerver/tests/test_decorators.py index f20da1ea75661..bbc82ccc3bb72 100644 --- a/zerver/tests/test_decorators.py +++ b/zerver/tests/test_decorators.py @@ -1,57 +1,81 @@ import base64 -from unittest import mock -import re import os +import re from collections import defaultdict - from typing import Any, Dict, Iterable, List, Optional, Tuple +from unittest import mock -from django.test import TestCase -from django.http import HttpResponse, HttpRequest +import ujson from django.conf import settings +from django.http import HttpRequest, HttpResponse +from django.test import TestCase -from zerver.forms import OurAuthenticationForm -from zerver.lib.actions import do_deactivate_realm, do_deactivate_user, \ - do_reactivate_user, do_reactivate_realm, do_set_realm_property -from zerver.lib.exceptions import JsonableError, InvalidAPIKeyError, InvalidAPIKeyFormatError -from zerver.lib.initial_password import initial_password -from zerver.lib.test_helpers import ( - HostRequestMock, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.lib.response import json_response, json_success -from zerver.lib.users import get_api_key -from zerver.lib.user_agent import parse_user_agent -from zerver.lib.utils import generate_api_key, has_api_key_format -from zerver.lib.request import \ - REQ, has_request_variables, RequestVariableMissingError, \ - RequestVariableConversionError, RequestConfusingParmsError -from zerver.lib.webhooks.common import UnexpectedWebhookEventType from zerver.decorator import ( api_key_only_webhook_view, + authenticate_notify, authenticated_json_view, authenticated_rest_api_view, authenticated_uploads_api_view, - authenticate_notify, cachify, - get_client_name, internal_notify_view, is_local_addr, - rate_limit, validate_api_key, + cachify, + get_client_name, + internal_notify_view, + is_local_addr, + rate_limit, return_success_on_head_request, + validate_api_key, zulip_login_required, ) -from zerver.lib.cache import ignore_unhashable_lru_cache, dict_to_items_tuple, items_tuple_to_dict +from zerver.forms import OurAuthenticationForm +from zerver.lib.actions import ( + do_deactivate_realm, + do_deactivate_user, + do_reactivate_realm, + do_reactivate_user, + do_set_realm_property, +) +from zerver.lib.cache import dict_to_items_tuple, ignore_unhashable_lru_cache, items_tuple_to_dict +from zerver.lib.exceptions import InvalidAPIKeyError, InvalidAPIKeyFormatError, JsonableError +from zerver.lib.initial_password import initial_password +from zerver.lib.request import ( + REQ, + RequestConfusingParmsError, + RequestVariableConversionError, + RequestVariableMissingError, + has_request_variables, +) +from zerver.lib.response import json_response, json_success +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import HostRequestMock +from zerver.lib.user_agent import parse_user_agent +from zerver.lib.users import get_api_key +from zerver.lib.utils import generate_api_key, has_api_key_format from zerver.lib.validator import ( - check_string, check_dict, check_dict_only, check_bool, check_float, check_int, check_list, Validator, - check_variable_type, equals, check_none_or, check_url, check_short_string, - check_string_fixed_length, check_capped_string, check_color, to_non_negative_int, - check_string_or_int_list, check_string_or_int, check_int_in, check_string_in, + Validator, + check_bool, + check_capped_string, + check_color, + check_dict, + check_dict_only, + check_float, + check_int, + check_int_in, + check_list, + check_none_or, + check_short_string, + check_string, + check_string_fixed_length, + check_string_in, + check_string_or_int, + check_string_or_int_list, + check_url, + check_variable_type, + equals, + to_non_negative_int, to_positive_or_allowed_int, ) -from zerver.models import \ - get_realm, get_user, UserProfile, Realm +from zerver.lib.webhooks.common import UnexpectedWebhookEventType +from zerver.models import Realm, UserProfile, get_realm, get_user -import ujson class DecoratorTestCase(TestCase): def test_get_client_name(self) -> None: diff --git a/zerver/tests/test_digest.py b/zerver/tests/test_digest.py index 6f634d676580e..cc3d3fb124dec 100644 --- a/zerver/tests/test_digest.py +++ b/zerver/tests/test_digest.py @@ -1,20 +1,34 @@ import datetime -from unittest import mock import time from typing import List +from unittest import mock from django.test import override_settings from django.utils.timezone import now as timezone_now from confirmation.models import one_click_unsubscribe_link from zerver.lib.actions import do_create_user -from zerver.lib.digest import gather_new_streams, handle_digest_email, enqueue_emails, \ - exclude_subscription_modified_streams +from zerver.lib.digest import ( + enqueue_emails, + exclude_subscription_modified_streams, + gather_new_streams, + handle_digest_email, +) from zerver.lib.streams import create_stream_if_needed from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import queries_captured -from zerver.models import get_client, get_realm, flush_per_request_caches, \ - Realm, Message, UserActivity, UserProfile, RealmAuditLog, get_stream +from zerver.models import ( + Message, + Realm, + RealmAuditLog, + UserActivity, + UserProfile, + flush_per_request_caches, + get_client, + get_realm, + get_stream, +) + class TestDigestEmailMessages(ZulipTestCase): diff --git a/zerver/tests/test_docs.py b/zerver/tests/test_docs.py index ee86e4041f7ae..e6c401510fcea 100644 --- a/zerver/tests/test_docs.py +++ b/zerver/tests/test_docs.py @@ -1,12 +1,12 @@ import os -import ujson +from typing import Any, Dict, List from unittest import mock from urllib.parse import urlsplit +import ujson from django.conf import settings -from django.test import TestCase, override_settings from django.http import HttpResponse -from typing import Any, Dict, List +from django.test import TestCase, override_settings from zerver.lib.integrations import INTEGRATIONS from zerver.lib.test_classes import ZulipTestCase @@ -14,9 +14,8 @@ from zerver.lib.test_runner import slow from zerver.lib.utils import split_by from zerver.models import Realm, get_realm -from zerver.views.documentation import ( - add_api_uri_context, -) +from zerver.views.documentation import add_api_uri_context + class DocPageTest(ZulipTestCase): def get_doc(self, url: str, subdomain: str) -> HttpResponse: diff --git a/zerver/tests/test_email_change.py b/zerver/tests/test_email_change.py index 6b8685774cae3..651dee7ed9e44 100644 --- a/zerver/tests/test_email_change.py +++ b/zerver/tests/test_email_change.py @@ -1,17 +1,22 @@ import datetime -from email.utils import parseaddr import re +from email.utils import parseaddr from django.core import mail from django.utils.timezone import now -from confirmation.models import Confirmation, generate_key, confirmation_url -from zerver.lib.actions import do_start_email_change_process, do_set_realm_property -from zerver.lib.test_classes import ( - ZulipTestCase, +from confirmation.models import Confirmation, confirmation_url, generate_key +from zerver.lib.actions import do_set_realm_property, do_start_email_change_process +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import ( + EmailChangeStatus, + Realm, + UserProfile, + get_realm, + get_user, + get_user_by_delivery_email, + get_user_profile_by_id, ) -from zerver.models import get_user_by_delivery_email, EmailChangeStatus, get_realm, \ - Realm, UserProfile, get_user, get_user_profile_by_id class EmailChangeTestCase(ZulipTestCase): diff --git a/zerver/tests/test_email_log.py b/zerver/tests/test_email_log.py index a720edf058a6d..b21f9fcb6639e 100644 --- a/zerver/tests/test_email_log.py +++ b/zerver/tests/test_email_log.py @@ -1,10 +1,12 @@ import os from unittest import mock + from django.conf import settings from zerver.lib.test_classes import ZulipTestCase from zproject.email_backends import get_forward_address + class EmailLogTest(ZulipTestCase): def test_generate_and_clear_email_log(self) -> None: with self.settings(EMAIL_BACKEND='zproject.email_backends.EmailLogBackEnd'), \ diff --git a/zerver/tests/test_email_mirror.py b/zerver/tests/test_email_mirror.py index c4205ff9bb0a9..d516807b6925e 100644 --- a/zerver/tests/test_email_mirror.py +++ b/zerver/tests/test_email_mirror.py @@ -1,63 +1,50 @@ +import os import subprocess +from email import message_from_string +from email.mime.image import MIMEImage +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from typing import Any, Callable, Dict, Mapping, Optional +from unittest import mock +import ujson +from django.conf import settings from django.http import HttpResponse -from zerver.lib.test_helpers import ( - most_recent_message, - most_recent_usermessage, -) - -from zerver.lib.test_classes import ( - ZulipTestCase, -) - -from zerver.models import ( - get_display_recipient, - get_realm, - get_stream, - get_system_bot, - MissedMessageEmailAddress, - Recipient, - UserProfile, -) - -from zerver.lib.actions import ensure_stream, do_deactivate_realm, do_deactivate_user - +from zerver.lib.actions import do_deactivate_realm, do_deactivate_user, ensure_stream from zerver.lib.email_mirror import ( - process_message, - process_missed_message, + ZulipEmailForwardError, create_missed_message_address, + filter_footer, get_missed_message_token_from_address, - strip_from_subject, is_forwarded, is_missed_message_address, - filter_footer, log_and_report, + process_message, + process_missed_message, redact_email_address, - ZulipEmailForwardError, + strip_from_subject, ) - from zerver.lib.email_mirror_helpers import ( decode_email_address, encode_email_address, get_email_gateway_message_string_from_address, ) - from zerver.lib.email_notifications import convert_html_to_markdown from zerver.lib.send_email import FromAddress +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import most_recent_message, most_recent_usermessage +from zerver.models import ( + MissedMessageEmailAddress, + Recipient, + UserProfile, + get_display_recipient, + get_realm, + get_stream, + get_system_bot, +) from zerver.worker.queue_processors import MirrorWorker -from email import message_from_string -from email.mime.text import MIMEText -from email.mime.image import MIMEImage -from email.mime.multipart import MIMEMultipart - -import ujson -from unittest import mock -import os -from django.conf import settings - -from typing import Any, Callable, Dict, Mapping, Optional class TestEncodeDecode(ZulipTestCase): def _assert_options(self, options: Dict[str, bool], show_sender: bool=False, diff --git a/zerver/tests/test_email_notifications.py b/zerver/tests/test_email_notifications.py index b8720dd3804a0..bbb134c7cde17 100644 --- a/zerver/tests/test_email_notifications.py +++ b/zerver/tests/test_email_notifications.py @@ -1,27 +1,27 @@ -import ldap import random import re -import ujson +from email.utils import formataddr +from typing import List, Optional +from unittest.mock import patch +import ldap +import ujson from django.conf import settings from django.core import mail from django.test import override_settings from django_auth_ldap.config import LDAPSearch -from email.utils import formataddr -from unittest.mock import patch -from typing import List, Optional -from zerver.lib.email_notifications import fix_emojis, handle_missedmessage_emails, \ - enqueue_welcome_emails, relative_to_full_url from zerver.lib.actions import do_change_notification_settings, do_change_user_role -from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.send_email import FromAddress, send_custom_email -from zerver.models import ( - get_realm, - get_stream, - UserProfile, - ScheduledEmail, +from zerver.lib.email_notifications import ( + enqueue_welcome_emails, + fix_emojis, + handle_missedmessage_emails, + relative_to_full_url, ) +from zerver.lib.send_email import FromAddress, send_custom_email +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import ScheduledEmail, UserProfile, get_realm, get_stream + class TestCustomEmails(ZulipTestCase): def test_send_custom_email_argument(self) -> None: diff --git a/zerver/tests/test_embedded_bot_system.py b/zerver/tests/test_embedded_bot_system.py index f70fdb5f8da12..768237544d9c2 100644 --- a/zerver/tests/test_embedded_bot_system.py +++ b/zerver/tests/test_embedded_bot_system.py @@ -1,13 +1,17 @@ from unittest.mock import patch +import ujson + from zerver.lib.bot_lib import EmbeddedBotQuitException from zerver.lib.test_classes import ZulipTestCase from zerver.models import ( - UserProfile, get_display_recipient, - get_service_profile, get_user, get_realm, + UserProfile, + get_display_recipient, + get_realm, + get_service_profile, + get_user, ) -import ujson class TestEmbeddedBotMessaging(ZulipTestCase): def setUp(self) -> None: diff --git a/zerver/tests/test_event_queue.py b/zerver/tests/test_event_queue.py index f80c8b1eedbd6..5a9cca4e83009 100644 --- a/zerver/tests/test_event_queue.py +++ b/zerver/tests/test_event_queue.py @@ -1,18 +1,24 @@ -from unittest import mock import time -import ujson +from typing import Any, Callable, Dict, Tuple +from unittest import mock +import ujson from django.http import HttpRequest, HttpResponse -from typing import Any, Callable, Dict, Tuple -from zerver.lib.actions import do_mute_topic, do_change_subscription_property +from zerver.lib.actions import do_change_subscription_property, do_mute_topic from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import POSTRequestMock from zerver.models import Recipient, Stream, Subscription, UserProfile, get_stream -from zerver.tornado.event_queue import maybe_enqueue_notifications, \ - allocate_client_descriptor, ClientDescriptor, \ - get_client_descriptor, missedmessage_hook, persistent_queue_filename -from zerver.tornado.views import get_events, cleanup_event_queue +from zerver.tornado.event_queue import ( + ClientDescriptor, + allocate_client_descriptor, + get_client_descriptor, + maybe_enqueue_notifications, + missedmessage_hook, + persistent_queue_filename, +) +from zerver.tornado.views import cleanup_event_queue, get_events + class MissedMessageNotificationsTest(ZulipTestCase): """Tests the logic for when missed-message notifications diff --git a/zerver/tests/test_events.py b/zerver/tests/test_events.py index 85f36b5a9cb72..f2a817df4fb62 100644 --- a/zerver/tests/test_events.py +++ b/zerver/tests/test_events.py @@ -1,29 +1,26 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/events-system.html for # high-level documentation on how this system works. -from typing import Any, Callable, Dict, List, Optional, Set, Tuple import copy import os import shutil import sys +import time +from io import StringIO +from typing import Any, Callable, Dict, List, Optional, Set, Tuple +from unittest import mock +import ujson from django.conf import settings from django.http import HttpRequest, HttpResponse from django.utils.timezone import now as timezone_now -from io import StringIO - -from zerver.models import ( - get_client, get_stream, get_realm, get_system_bot, - Message, RealmDomain, Recipient, UserMessage, UserPresence, UserProfile, - Realm, Subscription, Stream, flush_per_request_caches, UserGroup, Service, - Attachment, PreregistrationUser, get_user_by_delivery_email, MultiuseInvite, - RealmAuditLog, -) from zerver.lib.actions import ( - try_update_realm_custom_profile_field, + bulk_add_members_to_user_group, bulk_add_subscriptions, bulk_remove_subscriptions, check_add_realm_emoji, + check_add_user_group, + check_delete_user_group, check_send_message, check_send_typing_notification, do_add_alert_words, @@ -44,7 +41,6 @@ do_change_full_name, do_change_icon_source, do_change_logo_source, - do_change_user_role, do_change_notification_settings, do_change_plan_type, do_change_realm_domain, @@ -53,9 +49,10 @@ do_change_stream_post_policy, do_change_subscription_property, do_change_user_delivery_email, - do_create_user, + do_change_user_role, do_create_default_stream_group, do_create_multiuse_invite_link, + do_create_user, do_deactivate_stream, do_deactivate_user, do_delete_messages, @@ -78,10 +75,10 @@ do_revoke_user_invite, do_set_realm_authentication_methods, do_set_realm_message_editing, - do_set_realm_property, - do_set_user_display_setting, do_set_realm_notifications_stream, + do_set_realm_property, do_set_realm_signup_notifications_stream, + do_set_user_display_setting, do_set_zoom_token, do_unmute_topic, do_update_embedded_data, @@ -89,18 +86,16 @@ do_update_message_flags, do_update_outgoing_webhook_service, do_update_pointer, + do_update_user_custom_profile_data_if_changed, + do_update_user_group_description, + do_update_user_group_name, do_update_user_presence, do_update_user_status, log_event, lookup_default_stream_groups, notify_realm_custom_profile_fields, - check_add_user_group, - do_update_user_group_name, - do_update_user_group_description, - bulk_add_members_to_user_group, remove_members_from_user_group, - check_delete_user_group, - do_update_user_custom_profile_data_if_changed, + try_update_realm_custom_profile_field, ) from zerver.lib.bugdown import MentionData from zerver.lib.events import ( @@ -110,36 +105,65 @@ post_process_state, ) from zerver.lib.message import ( + MessageDict, + UnreadMessagesResult, aggregate_unread_data, apply_unread_message_event, get_raw_unread_data, render_markdown, - MessageDict, - UnreadMessagesResult, ) -from zerver.lib.test_helpers import POSTRequestMock, get_subscription, \ - get_test_image_file, stub_event_queue_user_events, queries_captured, \ - create_dummy_file, stdout_suppressed, reset_emails_in_zulip_realm -from zerver.lib.test_classes import ( - ZulipTestCase, +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import ( + POSTRequestMock, + create_dummy_file, + get_subscription, + get_test_image_file, + queries_captured, + reset_emails_in_zulip_realm, + stdout_suppressed, + stub_event_queue_user_events, ) from zerver.lib.test_runner import slow -from zerver.lib.topic import ( - ORIG_TOPIC, - TOPIC_NAME, - TOPIC_LINKS, -) -from zerver.lib.topic_mutes import ( - add_topic_mute, -) +from zerver.lib.topic import ORIG_TOPIC, TOPIC_LINKS, TOPIC_NAME +from zerver.lib.topic_mutes import add_topic_mute +from zerver.lib.users import get_api_key from zerver.lib.validator import ( - check_bool, check_dict, check_dict_only, check_float, check_int, check_list, check_string, - equals, check_none_or, Validator, check_url, check_int_in, + Validator, + check_bool, + check_dict, + check_dict_only, + check_float, + check_int, + check_int_in, + check_list, + check_none_or, + check_string, + check_url, + equals, +) +from zerver.models import ( + Attachment, + Message, + MultiuseInvite, + PreregistrationUser, + Realm, + RealmAuditLog, + RealmDomain, + Recipient, + Service, + Stream, + Subscription, + UserGroup, + UserMessage, + UserPresence, + UserProfile, + flush_per_request_caches, + get_client, + get_realm, + get_stream, + get_system_bot, + get_user_by_delivery_email, ) -from zerver.lib.users import get_api_key - -from zerver.views.events_register import _default_all_public_streams, _default_narrow - from zerver.tornado.event_queue import ( allocate_client_descriptor, clear_client_event_queues_for_testing, @@ -147,10 +171,7 @@ process_message_event, ) from zerver.tornado.views import get_events - -from unittest import mock -import time -import ujson +from zerver.views.events_register import _default_all_public_streams, _default_narrow class LogEventsTest(ZulipTestCase): diff --git a/zerver/tests/test_external.py b/zerver/tests/test_external.py index 38a1886f34b85..41273b6439c3f 100644 --- a/zerver/tests/test_external.py +++ b/zerver/tests/test_external.py @@ -1,28 +1,22 @@ +import time +from unittest import mock + +import DNS from django.conf import settings from django.core.exceptions import ValidationError from django.http import HttpResponse from zerver.forms import email_is_not_mit_mailing_list - from zerver.lib.rate_limiter import ( - add_ratelimit_rule, - remove_ratelimit_rule, RateLimitedUser, RateLimiterLockingException, + add_ratelimit_rule, + remove_ratelimit_rule, ) +from zerver.lib.test_classes import ZulipTestCase from zerver.lib.zephyr import compute_mit_user_fullname +from zerver.models import UserProfile -from zerver.lib.test_classes import ( - ZulipTestCase, -) - -from zerver.models import ( - UserProfile, -) - -import DNS -from unittest import mock -import time class MITNameTest(ZulipTestCase): def test_valid_hesiod(self) -> None: diff --git a/zerver/tests/test_gitter_importer.py b/zerver/tests/test_gitter_importer.py index a50ba1981dc4c..89c217844e5a9 100644 --- a/zerver/tests/test_gitter_importer.py +++ b/zerver/tests/test_gitter_importer.py @@ -1,24 +1,15 @@ -from zerver.lib.import_realm import ( - do_import_realm, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.models import ( - get_realm, - UserProfile, - Message, -) -from zerver.data_import.gitter import ( - do_convert_data, - get_usermentions, -) - -import ujson import logging import os -from unittest import mock from typing import Any +from unittest import mock + +import ujson + +from zerver.data_import.gitter import do_convert_data, get_usermentions +from zerver.lib.import_realm import do_import_realm +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import Message, UserProfile, get_realm + class GitterImporter(ZulipTestCase): logger = logging.getLogger() diff --git a/zerver/tests/test_hipchat_importer.py b/zerver/tests/test_hipchat_importer.py index 709e2a2191295..55657729f18ef 100644 --- a/zerver/tests/test_hipchat_importer.py +++ b/zerver/tests/test_hipchat_importer.py @@ -1,18 +1,10 @@ -from zerver.data_import.hipchat import ( - get_hipchat_sender_id, -) -from zerver.data_import.hipchat_user import ( - UserHandler, -) -from zerver.data_import.sequencer import ( - IdMapper, -) - -from zerver.lib.test_classes import ( - ZulipTestCase, -) from typing import Any, Dict +from zerver.data_import.hipchat import get_hipchat_sender_id +from zerver.data_import.hipchat_user import UserHandler +from zerver.data_import.sequencer import IdMapper +from zerver.lib.test_classes import ZulipTestCase + class HipChatImporter(ZulipTestCase): def test_sender_ids(self) -> None: diff --git a/zerver/tests/test_home.py b/zerver/tests/test_home.py index b1fc3b62b5059..e1b00d40c458d 100644 --- a/zerver/tests/test_home.py +++ b/zerver/tests/test_home.py @@ -1,34 +1,38 @@ import calendar +import urllib from datetime import timedelta +from typing import Any, Dict +from unittest.mock import patch + import lxml.html import ujson - from django.conf import settings from django.http import HttpResponse from django.utils.timezone import now as timezone_now -from unittest.mock import patch -import urllib -from typing import Any, Dict -from zerver.lib.actions import ( - do_create_user, do_change_logo_source, -) + +from corporate.models import Customer, CustomerPlan +from zerver.lib.actions import do_change_logo_source, do_create_user from zerver.lib.events import add_realm_logo_fields -from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.test_helpers import ( - queries_captured, get_user_messages, -) from zerver.lib.soft_deactivation import do_soft_deactivate_users +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import get_user_messages, queries_captured from zerver.lib.test_runner import slow from zerver.lib.users import compute_show_invites_and_add_streams from zerver.models import ( - get_realm, get_stream, get_user, UserProfile, - flush_per_request_caches, DefaultStream, Realm, - get_system_bot, UserActivity, + DefaultStream, + Realm, + UserActivity, + UserProfile, + flush_per_request_caches, + get_realm, + get_stream, + get_system_bot, + get_user, ) from zerver.views.home import compute_navbar_logo_url, get_furthest_read_time -from corporate.models import Customer, CustomerPlan from zerver.worker.queue_processors import UserActivityWorker + class HomeTest(ZulipTestCase): def test_home(self) -> None: diff --git a/zerver/tests/test_hotspots.py b/zerver/tests/test_hotspots.py index c516307d04483..59123353b4d62 100644 --- a/zerver/tests/test_hotspots.py +++ b/zerver/tests/test_hotspots.py @@ -1,9 +1,10 @@ -from zerver.lib.actions import do_mark_hotspot_as_read, do_create_user +import ujson + +from zerver.lib.actions import do_create_user, do_mark_hotspot_as_read from zerver.lib.hotspots import ALL_HOTSPOTS, get_next_hotspots from zerver.lib.test_classes import ZulipTestCase -from zerver.models import UserProfile, UserHotspot, get_realm +from zerver.models import UserHotspot, UserProfile, get_realm -import ujson # Splitting this out, since I imagine this will eventually have most of the # complicated hotspots logic. diff --git a/zerver/tests/test_i18n.py b/zerver/tests/test_i18n.py index 6c550da04392c..804998b3fd56b 100644 --- a/zerver/tests/test_i18n.py +++ b/zerver/tests/test_i18n.py @@ -1,20 +1,19 @@ +from http.cookies import SimpleCookie from typing import Any - from unittest import mock + import ujson -from django.test import TestCase -from django.utils import translation from django.conf import settings -from django.http import HttpResponse from django.core import mail -from http.cookies import SimpleCookie +from django.http import HttpResponse +from django.test import TestCase +from django.utils import translation -from zerver.models import get_realm_stream -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.management.commands import makemessages from zerver.lib.email_notifications import enqueue_welcome_emails +from zerver.lib.test_classes import ZulipTestCase +from zerver.management.commands import makemessages +from zerver.models import get_realm_stream + class EmailTranslationTestCase(ZulipTestCase): def test_email_translation(self) -> None: diff --git a/zerver/tests/test_import_export.py b/zerver/tests/test_import_export.py index 809c2b345cc01..0fdacd442ad37 100644 --- a/zerver/tests/test_import_export.py +++ b/zerver/tests/test_import_export.py @@ -1,95 +1,68 @@ -from django.conf import settings - import os -import ujson - +from typing import Any, Callable, Dict, FrozenSet, List, Optional, Set, Tuple from unittest.mock import patch -from typing import Any, Dict, List, Set, Optional, Tuple, Callable, \ - FrozenSet + +import ujson +from django.conf import settings from django.db.models import Q from django.utils.timezone import now as timezone_now -from zerver.lib.export import ( - do_export_realm, - export_usermessages_batch, - do_export_user, -) -from zerver.lib.import_realm import ( - do_import_realm, - get_incoming_message_ids, -) -from zerver.lib.avatar_hash import ( - user_avatar_path, -) -from zerver.lib.upload import ( - claim_attachment, - upload_message_file, - upload_emoji_image, - upload_avatar_image, -) from zerver.lib import upload - -from zerver.lib.utils import ( - query_chunker, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.lib.test_helpers import ( - get_test_image_file, - use_s3_backend, - create_s3_buckets, -) - -from zerver.lib.topic_mutes import ( - add_topic_mute, -) -from zerver.lib.bot_lib import ( - StateHandler, -) -from zerver.lib.bot_config import ( - set_bot_config, -) from zerver.lib.actions import ( - do_create_user, do_add_reaction, do_change_icon_source, do_change_logo_source, - do_update_user_presence, do_change_plan_type, + do_create_user, + do_update_user_presence, ) +from zerver.lib.avatar_hash import user_avatar_path +from zerver.lib.bot_config import set_bot_config +from zerver.lib.bot_lib import StateHandler +from zerver.lib.export import do_export_realm, do_export_user, export_usermessages_batch +from zerver.lib.import_realm import do_import_realm, get_incoming_message_ids from zerver.lib.streams import create_stream_if_needed +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import create_s3_buckets, get_test_image_file, use_s3_backend from zerver.lib.test_runner import slow - +from zerver.lib.topic_mutes import add_topic_mute +from zerver.lib.upload import ( + claim_attachment, + upload_avatar_image, + upload_emoji_image, + upload_message_file, +) +from zerver.lib.utils import query_chunker from zerver.models import ( - Message, - Realm, - Stream, - UserProfile, - Subscription, Attachment, - RealmEmoji, - Reaction, - Recipient, - UserMessage, + BotConfigData, + BotStorageData, CustomProfileField, CustomProfileFieldValue, - RealmAuditLog, Huddle, - UserHotspot, + Message, MutedTopic, + Reaction, + Realm, + RealmAuditLog, + RealmEmoji, + Recipient, + Stream, + Subscription, UserGroup, UserGroupMembership, + UserHotspot, + UserMessage, UserPresence, - BotStorageData, - BotConfigData, + UserProfile, get_active_streams, get_client, + get_huddle_hash, get_realm, get_stream, - get_huddle_hash, ) + class QueryUtilTest(ZulipTestCase): def _create_messages(self) -> None: for name in ['cordelia', 'hamlet', 'iago']: diff --git a/zerver/tests/test_integrations.py b/zerver/tests/test_integrations.py index ae4b0e136a6a8..ff0bb2b7d4ff7 100644 --- a/zerver/tests/test_integrations.py +++ b/zerver/tests/test_integrations.py @@ -1,7 +1,15 @@ -from zerver.lib.test_classes import ZulipTestCase from zerver.lib.integrations import ( - split_fixture_path, get_fixture_and_image_paths, INTEGRATIONS, ScreenshotConfig, WebhookIntegration, - DOC_SCREENSHOT_CONFIG, WEBHOOK_INTEGRATIONS, NO_SCREENSHOT_WEBHOOKS) + DOC_SCREENSHOT_CONFIG, + INTEGRATIONS, + NO_SCREENSHOT_WEBHOOKS, + WEBHOOK_INTEGRATIONS, + ScreenshotConfig, + WebhookIntegration, + get_fixture_and_image_paths, + split_fixture_path, +) +from zerver.lib.test_classes import ZulipTestCase + class IntegrationsTestCase(ZulipTestCase): diff --git a/zerver/tests/test_integrations_dev_panel.py b/zerver/tests/test_integrations_dev_panel.py index 3a6dd4528fbfa..7c7af870fab24 100644 --- a/zerver/tests/test_integrations_dev_panel.py +++ b/zerver/tests/test_integrations_dev_panel.py @@ -1,7 +1,10 @@ -import ujson from unittest.mock import MagicMock, patch + +import ujson + from zerver.lib.test_classes import ZulipTestCase -from zerver.models import get_user, get_realm, Message, Stream +from zerver.models import Message, Stream, get_realm, get_user + class TestIntegrationsDevPanel(ZulipTestCase): diff --git a/zerver/tests/test_internet.py b/zerver/tests/test_internet.py index 0fee3306e585d..6172c6f4c2c74 100644 --- a/zerver/tests/test_internet.py +++ b/zerver/tests/test_internet.py @@ -1,7 +1,8 @@ +import requests +import responses + from zerver.lib.test_classes import ZulipTestCase -import responses -import requests class ResponsesTest(ZulipTestCase): def test_responses(self) -> None: diff --git a/zerver/tests/test_legacy_subject.py b/zerver/tests/test_legacy_subject.py index 436a9de45297a..90702c7d69ae6 100644 --- a/zerver/tests/test_legacy_subject.py +++ b/zerver/tests/test_legacy_subject.py @@ -1,6 +1,5 @@ -from zerver.lib.test_classes import ( - ZulipTestCase, -) +from zerver.lib.test_classes import ZulipTestCase + class LegacySubjectTest(ZulipTestCase): def test_legacy_subject(self) -> None: diff --git a/zerver/tests/test_link_embed.py b/zerver/tests/test_link_embed.py index 14a1204bc5a73..c6b19a126a002 100644 --- a/zerver/tests/test_link_embed.py +++ b/zerver/tests/test_link_embed.py @@ -1,22 +1,20 @@ +from typing import Any, Callable, Dict, Optional from unittest import mock + import ujson -from typing import Any, Callable, Dict, Optional -from requests.exceptions import ConnectionError from django.test import override_settings from django.utils.html import escape +from requests.exceptions import ConnectionError -from zerver.models import Message, Realm, UserProfile from zerver.lib.actions import queue_json_publish +from zerver.lib.cache import NotFoundInCache, cache_set, preview_url_cache_key from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import MockPythonResponse -from zerver.worker.queue_processors import FetchLinksEmbedData -from zerver.lib.url_preview.preview import ( - get_link_embed_data, link_embed_data_from_cache) from zerver.lib.url_preview.oembed import get_oembed_data, strip_cdata -from zerver.lib.url_preview.parsers import ( - OpenGraphParser, GenericParser) -from zerver.lib.cache import cache_set, NotFoundInCache, preview_url_cache_key - +from zerver.lib.url_preview.parsers import GenericParser, OpenGraphParser +from zerver.lib.url_preview.preview import get_link_embed_data, link_embed_data_from_cache +from zerver.models import Message, Realm, UserProfile +from zerver.worker.queue_processors import FetchLinksEmbedData TEST_CACHES = { 'default': { diff --git a/zerver/tests/test_logging_handlers.py b/zerver/tests/test_logging_handlers.py index fd90ee3afdfb9..d3a266b7ac90b 100644 --- a/zerver/tests/test_logging_handlers.py +++ b/zerver/tests/test_logging_handlers.py @@ -1,18 +1,18 @@ import logging import sys +from functools import wraps +from types import TracebackType +from typing import Any, Callable, Dict, Iterator, NoReturn, Optional, Tuple, Type +from unittest.mock import MagicMock, patch from django.conf import settings from django.contrib.auth.models import AnonymousUser from django.http import HttpRequest from django.test import TestCase from django.utils.log import AdminEmailHandler -from functools import wraps -from unittest.mock import MagicMock, patch -from types import TracebackType -from typing import Any, Callable, Dict, Iterator, NoReturn, Optional, Tuple, Type -from zerver.lib.types import ViewFuncT from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.types import ViewFuncT from zerver.logging_handlers import AdminNotifyHandler captured_request: Optional[HttpRequest] = None diff --git a/zerver/tests/test_management_commands.py b/zerver/tests/test_management_commands.py index cb47a55667389..f0d9d7d26fd8c 100644 --- a/zerver/tests/test_management_commands.py +++ b/zerver/tests/test_management_commands.py @@ -3,24 +3,32 @@ import re from datetime import timedelta from email.utils import parseaddr +from typing import Any, Dict, List, Optional from unittest import mock -from unittest.mock import MagicMock, patch, call -from typing import List, Dict, Any, Optional +from unittest.mock import MagicMock, call, patch from django.conf import settings from django.core.management import call_command from django.test import TestCase, override_settings -from zerver.lib.actions import do_create_user, do_add_reaction -from zerver.lib.management import ZulipBaseCommand, CommandError, check_config -from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.test_helpers import stdout_suppressed -from zerver.lib.test_runner import slow -from zerver.models import Recipient, get_user_profile_by_email, get_stream from django.utils.timezone import now as timezone_now -from zerver.lib.test_helpers import most_recent_message -from zerver.models import get_realm, UserProfile, Realm, Reaction, Message from confirmation.models import RealmCreationKey, generate_realm_creation_url +from zerver.lib.actions import do_add_reaction, do_create_user +from zerver.lib.management import CommandError, ZulipBaseCommand, check_config +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import most_recent_message, stdout_suppressed +from zerver.lib.test_runner import slow +from zerver.models import ( + Message, + Reaction, + Realm, + Recipient, + UserProfile, + get_realm, + get_stream, + get_user_profile_by_email, +) + class TestCheckConfig(ZulipTestCase): def test_check_config(self) -> None: diff --git a/zerver/tests/test_mattermost_importer.py b/zerver/tests/test_mattermost_importer.py index ff0978b3cd546..f98784b4d0e47 100644 --- a/zerver/tests/test_mattermost_importer.py +++ b/zerver/tests/test_mattermost_importer.py @@ -1,27 +1,34 @@ -import os -import ujson import filecmp import logging +import os +from typing import Any, Dict, List -from typing import Dict, Any, List +import ujson -from zerver.lib.emoji import name_to_codepoint -from zerver.lib.import_realm import ( - do_import_realm, -) -from zerver.lib.test_classes import ( - ZulipTestCase, +from zerver.data_import.import_util import SubscriberHandler +from zerver.data_import.mattermost import ( + build_reactions, + check_user_in_team, + convert_channel_data, + convert_huddle_data, + convert_user_data, + create_username_to_user_mapping, + do_convert_data, + generate_huddle_name, + get_mentioned_user_ids, + label_mirror_dummy_users, + mattermost_data_file_to_dict, + process_user, + reset_mirror_dummy_users, + write_emoticon_data, ) - from zerver.data_import.mattermost_user import UserHandler -from zerver.data_import.mattermost import mattermost_data_file_to_dict, process_user, convert_user_data, \ - create_username_to_user_mapping, label_mirror_dummy_users, reset_mirror_dummy_users, \ - convert_channel_data, write_emoticon_data, get_mentioned_user_ids, check_user_in_team, \ - build_reactions, do_convert_data, convert_huddle_data, \ - generate_huddle_name from zerver.data_import.sequencer import IdMapper -from zerver.data_import.import_util import SubscriberHandler -from zerver.models import Reaction, UserProfile, Message, get_realm, get_user, Recipient +from zerver.lib.emoji import name_to_codepoint +from zerver.lib.import_realm import do_import_realm +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import Message, Reaction, Recipient, UserProfile, get_realm, get_user + class MatterMostImporter(ZulipTestCase): logger = logging.getLogger() diff --git a/zerver/tests/test_message_edit_notifications.py b/zerver/tests/test_message_edit_notifications.py index c2f882b2a7c9b..00800c1e78e83 100644 --- a/zerver/tests/test_message_edit_notifications.py +++ b/zerver/tests/test_message_edit_notifications.py @@ -1,25 +1,13 @@ from typing import Any, Dict, Mapping, Union - from unittest import mock from django.utils.timezone import now as timezone_now -from zerver.lib.actions import ( - get_client, -) - -from zerver.lib.test_classes import ( - ZulipTestCase, -) - -from zerver.models import ( - Subscription, - UserPresence, -) +from zerver.lib.actions import get_client +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import Subscription, UserPresence +from zerver.tornado.event_queue import maybe_enqueue_notifications -from zerver.tornado.event_queue import ( - maybe_enqueue_notifications, -) class EditMessageSideEffectsTest(ZulipTestCase): def _assert_update_does_not_notify_anybody(self, message_id: int, content: str) -> None: diff --git a/zerver/tests/test_messages.py b/zerver/tests/test_messages.py index 6a110fd8b8719..bfec18fcbe4e4 100644 --- a/zerver/tests/test_messages.py +++ b/zerver/tests/test_messages.py @@ -1,15 +1,22 @@ +import datetime +import time +from collections import namedtuple +from operator import itemgetter +from typing import Any, Dict, List, Set, Tuple, Union +from unittest import mock + +import ujson +from django.conf import settings from django.db import IntegrityError from django.db.models import Q -from django.conf import settings from django.http import HttpResponse from django.test import TestCase, override_settings from django.utils.timezone import now as timezone_now -from zerver.lib import bugdown +from analytics.lib.counts import COUNT_STATS +from analytics.models import RealmCount from zerver.decorator import JsonableError -from zerver.lib.test_runner import slow -from zerver.lib.addressee import Addressee - +from zerver.lib import bugdown from zerver.lib.actions import ( check_message, check_send_stream_message, @@ -22,8 +29,8 @@ do_create_user, do_deactivate_user, do_send_messages, - do_update_message, do_set_realm_property, + do_update_message, extract_private_recipients, extract_stream_indicator, gather_subscriptions_helper, @@ -40,18 +47,9 @@ internal_send_stream_message_by_name, send_rate_limited_pm_notification_to_bot_owner, ) - -from zerver.lib.cache import ( - cache_delete, - get_stream_cache_key, - to_dict_cache_key_id, -) - - -from zerver.lib.create_user import ( - create_user_profile, -) - +from zerver.lib.addressee import Addressee +from zerver.lib.cache import cache_delete, get_stream_cache_key, to_dict_cache_key_id +from zerver.lib.create_user import create_user_profile from zerver.lib.message import ( MessageDict, bulk_access_messages, @@ -64,7 +62,13 @@ sew_messages_and_reactions, update_first_visible_message_id, ) - +from zerver.lib.soft_deactivation import ( + add_missing_messages, + do_soft_activate_users, + do_soft_deactivate_users, + reactivate_user_if_soft_deactivated, +) +from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import ( get_subscription, get_user_messages, @@ -75,55 +79,42 @@ queries_captured, reset_emails_in_zulip_realm, ) - -from zerver.lib.test_classes import ( - ZulipTestCase, -) - -from zerver.lib.topic import ( - LEGACY_PREV_TOPIC, - DB_TOPIC_NAME, - TOPIC_LINKS, - TOPIC_NAME, -) - -from zerver.lib.types import DisplayRecipientT, UserDisplayRecipient -from zerver.lib.soft_deactivation import ( - add_missing_messages, - do_soft_activate_users, - do_soft_deactivate_users, - reactivate_user_if_soft_deactivated, -) - -from zerver.models import ( - MAX_MESSAGE_LENGTH, MAX_TOPIC_NAME_LENGTH, - Message, Realm, Recipient, Stream, UserMessage, UserProfile, Attachment, - RealmAuditLog, RealmDomain, get_realm, UserPresence, Subscription, - get_stream, get_system_bot, get_user, Reaction, - flush_per_request_caches, ScheduledMessage, get_huddle_recipient, - bulk_get_huddle_user_ids, get_huddle_user_ids, - get_display_recipient, RealmFilter, -) - - +from zerver.lib.test_runner import slow from zerver.lib.timestamp import convert_to_UTC, datetime_to_timestamp from zerver.lib.timezone import get_timezone +from zerver.lib.topic import DB_TOPIC_NAME, LEGACY_PREV_TOPIC, TOPIC_LINKS, TOPIC_NAME +from zerver.lib.types import DisplayRecipientT, UserDisplayRecipient from zerver.lib.upload import create_attachment from zerver.lib.url_encoding import near_message_url +from zerver.models import ( + MAX_MESSAGE_LENGTH, + MAX_TOPIC_NAME_LENGTH, + Attachment, + Message, + Reaction, + Realm, + RealmAuditLog, + RealmDomain, + RealmFilter, + Recipient, + ScheduledMessage, + Stream, + Subscription, + UserMessage, + UserPresence, + UserProfile, + bulk_get_huddle_user_ids, + flush_per_request_caches, + get_display_recipient, + get_huddle_recipient, + get_huddle_user_ids, + get_realm, + get_stream, + get_system_bot, + get_user, +) +from zerver.views.messages import InvalidMirrorInput, create_mirrored_message_users -from zerver.views.messages import create_mirrored_message_users, InvalidMirrorInput - -from analytics.lib.counts import COUNT_STATS -from analytics.models import RealmCount - -import datetime -from unittest import mock -from operator import itemgetter -import time -import ujson -from typing import Any, Dict, List, Set, Union, Tuple - -from collections import namedtuple class MiscMessageTest(ZulipTestCase): def test_get_last_message_id(self) -> None: diff --git a/zerver/tests/test_middleware.py b/zerver/tests/test_middleware.py index 1e5c0a440f0f6..6506b73f0440c 100644 --- a/zerver/tests/test_middleware.py +++ b/zerver/tests/test_middleware.py @@ -1,13 +1,15 @@ import time from typing import List +from unittest.mock import patch from bs4 import BeautifulSoup -from unittest.mock import patch + from zerver.lib.realm_icon import get_realm_icon_url from zerver.lib.test_classes import ZulipTestCase from zerver.middleware import is_slow_query, write_log_line from zerver.models import get_realm + class SlowQueryTest(ZulipTestCase): SLOW_QUERY_TIME = 10 log_data = {'extra': '[transport=websocket]', diff --git a/zerver/tests/test_migrations.py b/zerver/tests/test_migrations.py index 7382f613a26c2..a83f190b9a5d0 100644 --- a/zerver/tests/test_migrations.py +++ b/zerver/tests/test_migrations.py @@ -4,11 +4,10 @@ # You can also read # https://www.caktusgroup.com/blog/2016/02/02/writing-unit-tests-django-migrations/ # to get a tutorial on the framework that inspired this feature. +from django.db.migrations.state import StateApps from zerver.lib.test_classes import MigrationsTestCase from zerver.lib.test_helpers import use_db_models -from django.db.migrations.state import StateApps - from zerver.models import get_stream # Important note: These tests are very expensive, and details of diff --git a/zerver/tests/test_muting.py b/zerver/tests/test_muting.py index 5a64ee928e314..91d72ff386885 100644 --- a/zerver/tests/test_muting.py +++ b/zerver/tests/test_muting.py @@ -1,23 +1,19 @@ -from django.utils.timezone import now as timezone_now -from datetime import timedelta, datetime, timezone +from datetime import datetime, timedelta, timezone from typing import Any, Dict from unittest import mock -from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.stream_topic import StreamTopicTarget - -from zerver.models import ( - get_stream, - UserProfile, - MutedTopic, -) +from django.utils.timezone import now as timezone_now +from zerver.lib.stream_topic import StreamTopicTarget +from zerver.lib.test_classes import ZulipTestCase from zerver.lib.topic_mutes import ( add_topic_mute, get_topic_mutes, remove_topic_mute, topic_is_muted, ) +from zerver.models import MutedTopic, UserProfile, get_stream + class MutedTopicsTests(ZulipTestCase): def test_user_ids_muting_topic(self) -> None: diff --git a/zerver/tests/test_narrow.py b/zerver/tests/test_narrow.py index a118e444911fe..0eaf11bae6fa7 100644 --- a/zerver/tests/test_narrow.py +++ b/zerver/tests/test_narrow.py @@ -1,57 +1,47 @@ +import os +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union +from unittest import mock + +import ujson from django.db import connection from django.test import TestCase, override_settings -from sqlalchemy.sql import ( - and_, select, column, table, -) +from sqlalchemy.sql import and_, column, select, table from sqlalchemy.sql.elements import ClauseElement -from zerver.models import ( - Realm, Subscription, Recipient, Stream, - get_display_recipient, get_realm, get_stream, - UserMessage, Message, -) -from zerver.lib.actions import ( - do_set_realm_property, - do_deactivate_user, -) -from zerver.lib.message import ( - MessageDict, -) -from zerver.lib.narrow import ( - build_narrow_filter, - is_web_public_compatible, -) +from zerver.lib.actions import do_deactivate_user, do_set_realm_property +from zerver.lib.message import MessageDict +from zerver.lib.narrow import build_narrow_filter, is_web_public_compatible from zerver.lib.request import JsonableError from zerver.lib.sqlalchemy_utils import get_sqlalchemy_connection -from zerver.lib.test_helpers import ( - POSTRequestMock, - get_user_messages, queries_captured, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.lib.topic import ( - MATCH_TOPIC, - TOPIC_NAME, -) -from zerver.lib.topic_mutes import ( - set_topic_mutes, -) +from zerver.lib.streams import create_streams_if_needed +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import POSTRequestMock, get_user_messages, queries_captured +from zerver.lib.topic import MATCH_TOPIC, TOPIC_NAME +from zerver.lib.topic_mutes import set_topic_mutes from zerver.lib.types import DisplayRecipientT +from zerver.models import ( + Message, + Realm, + Recipient, + Stream, + Subscription, + UserMessage, + get_display_recipient, + get_realm, + get_stream, +) from zerver.views.messages import ( + LARGER_THAN_MAX_MESSAGE_ID, + BadNarrowOperator, + NarrowBuilder, + Query, exclude_muting_conditions, - get_messages_backend, ok_to_include_history, - NarrowBuilder, BadNarrowOperator, Query, - post_process_limited_query, find_first_unread_anchor, - LARGER_THAN_MAX_MESSAGE_ID, + get_messages_backend, + ok_to_include_history, + post_process_limited_query, ) -from zerver.lib.streams import create_streams_if_needed -from typing import Dict, Mapping, List, Sequence, Tuple, Union, Any, Optional -from unittest import mock -import os -import ujson def get_sqlalchemy_sql(query: ClauseElement) -> str: dialect = get_sqlalchemy_connection().dialect diff --git a/zerver/tests/test_new_users.py b/zerver/tests/test_new_users.py index ec1baccb00fc3..9cc3778ab65f8 100644 --- a/zerver/tests/test_new_users.py +++ b/zerver/tests/test_new_users.py @@ -1,15 +1,17 @@ import datetime +from unittest import mock + from django.conf import settings from django.core import mail from django.test import override_settings -from zerver.lib.test_classes import ZulipTestCase -from zerver.signals import get_device_browser, get_device_os, JUST_CREATED_THRESHOLD -from zerver.lib.actions import notify_new_user, do_change_notification_settings -from zerver.models import Recipient, Stream, Realm +from zerver.lib.actions import do_change_notification_settings, notify_new_user from zerver.lib.initial_password import initial_password -from unittest import mock +from zerver.lib.test_classes import ZulipTestCase from zerver.lib.timezone import get_timezone +from zerver.models import Realm, Recipient, Stream +from zerver.signals import JUST_CREATED_THRESHOLD, get_device_browser, get_device_os + class SendLoginEmailTest(ZulipTestCase): """ diff --git a/zerver/tests/test_onboarding.py b/zerver/tests/test_onboarding.py index 4477257c20a7b..ace78fcd29025 100644 --- a/zerver/tests/test_onboarding.py +++ b/zerver/tests/test_onboarding.py @@ -1,8 +1,7 @@ -from zerver.models import Realm, UserProfile from zerver.lib.onboarding import create_if_missing_realm_internal_bots -from zerver.lib.test_classes import ( - ZulipTestCase, -) +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import Realm, UserProfile + class TestRealmInternalBotCreation(ZulipTestCase): def test_create_if_missing_realm_internal_bots(self) -> None: diff --git a/zerver/tests/test_openapi.py b/zerver/tests/test_openapi.py index 7a7f8fff1a9de..03dc02bdd0cf4 100644 --- a/zerver/tests/test_openapi.py +++ b/zerver/tests/test_openapi.py @@ -1,24 +1,43 @@ +import inspect import re import sys +from typing import ( + Any, + Callable, + Dict, + Iterable, + List, + Mapping, + Optional, + Sequence, + Set, + Tuple, + Union, +) from unittest import mock -import inspect -from typing import Dict, Any, Set, Union, List, Callable, Tuple, Optional, Iterable, Mapping, Sequence -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch from django.http import HttpResponse -import zerver.openapi.openapi as openapi -from zerver.openapi.markdown_extension import generate_curl_example, \ - render_curl_example, parse_language_and_options -from zerver.lib.request import _REQ +from zerver.lib.request import _REQ, arguments_map from zerver.lib.test_classes import ZulipTestCase +from zerver.openapi import openapi as openapi +from zerver.openapi.markdown_extension import ( + generate_curl_example, + parse_language_and_options, + render_curl_example, +) from zerver.openapi.openapi import ( - get_openapi_fixture, get_openapi_parameters, - validate_against_openapi_schema, to_python_type, - SchemaError, openapi_spec, get_openapi_paths, - OpenAPISpec, OPENAPI_SPEC_PATH, + OPENAPI_SPEC_PATH, + OpenAPISpec, + SchemaError, + get_openapi_fixture, + get_openapi_parameters, + get_openapi_paths, + openapi_spec, + to_python_type, + validate_against_openapi_schema, ) -from zerver.lib.request import arguments_map TEST_ENDPOINT = '/messages/{message_id}' TEST_METHOD = 'patch' @@ -525,7 +544,7 @@ def test_openapi_arguments(self) -> None: in code. """ - import zproject.urls as urlconf + from zproject import urls as urlconf # We loop through all the API patterns, looking in particular # for those using the rest_dispatch decorator; we then parse diff --git a/zerver/tests/test_outgoing_webhook_interfaces.py b/zerver/tests/test_outgoing_webhook_interfaces.py index 5e141ec8b0e45..75b8ecdc87377 100644 --- a/zerver/tests/test_outgoing_webhook_interfaces.py +++ b/zerver/tests/test_outgoing_webhook_interfaces.py @@ -1,25 +1,16 @@ -from typing import cast, Any, Dict - -from unittest import mock import json +from typing import Any, Dict, cast +from unittest import mock + import requests from zerver.lib.avatar import get_gravatar_url from zerver.lib.message import MessageDict -from zerver.lib.outgoing_webhook import ( - get_service_interface_class, - process_success_response, -) +from zerver.lib.outgoing_webhook import get_service_interface_class, process_success_response from zerver.lib.test_classes import ZulipTestCase from zerver.lib.timestamp import datetime_to_timestamp from zerver.lib.topic import TOPIC_NAME -from zerver.models import ( - get_realm, - get_stream, - get_user, - Message, - SLACK_INTERFACE, -) +from zerver.models import SLACK_INTERFACE, Message, get_realm, get_stream, get_user class TestGenericOutgoingWebhookService(ZulipTestCase): diff --git a/zerver/tests/test_outgoing_webhook_system.py b/zerver/tests/test_outgoing_webhook_system.py index aee46890aebea..b0c30cf3f48b8 100644 --- a/zerver/tests/test_outgoing_webhook_system.py +++ b/zerver/tests/test_outgoing_webhook_system.py @@ -1,33 +1,29 @@ -import ujson import logging -from unittest import mock -import requests - from typing import Any, Optional +from unittest import mock -from zerver.lib.actions import ( - do_create_user, -) +import requests +import ujson +from version import ZULIP_VERSION +from zerver.lib.actions import do_create_user from zerver.lib.outgoing_webhook import ( - do_rest_call, GenericOutgoingWebhookService, SlackOutgoingWebhookService, + do_rest_call, ) - from zerver.lib.test_classes import ZulipTestCase from zerver.lib.topic import TOPIC_NAME from zerver.lib.users import add_service from zerver.models import ( - get_display_recipient, - get_realm, - get_user, Recipient, Service, UserProfile, + get_display_recipient, + get_realm, + get_user, ) -from version import ZULIP_VERSION class ResponseMock: def __init__(self, status_code: int, content: Optional[Any]=None) -> None: diff --git a/zerver/tests/test_populate_db.py b/zerver/tests/test_populate_db.py index 31c946c050dd8..1111d7385894a 100644 --- a/zerver/tests/test_populate_db.py +++ b/zerver/tests/test_populate_db.py @@ -1,7 +1,8 @@ -from zilencer.management.commands.populate_db import choose_date_sent +from django.utils.timezone import timedelta as timezone_timedelta + from zerver.lib.test_classes import ZulipTestCase +from zilencer.management.commands.populate_db import choose_date_sent -from django.utils.timezone import timedelta as timezone_timedelta class TestChoosePubDate(ZulipTestCase): def test_choose_date_sent_large_tot_messages(self) -> None: diff --git a/zerver/tests/test_presence.py b/zerver/tests/test_presence.py index 9a16db32d6894..52cb221ee91e2 100644 --- a/zerver/tests/test_presence.py +++ b/zerver/tests/test_presence.py @@ -1,33 +1,26 @@ +import datetime from datetime import timedelta -from django.utils.timezone import now as timezone_now +from typing import Any, Dict from unittest import mock -from typing import Any, Dict +from django.utils.timezone import now as timezone_now + from zerver.lib.actions import do_deactivate_user -from zerver.lib.presence import ( - get_status_dict_by_realm, -) +from zerver.lib.presence import get_status_dict_by_realm from zerver.lib.statistics import seconds_usage_between -from zerver.lib.test_helpers import ( - make_client, - queries_captured, - reset_emails_in_zulip_realm, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import make_client, queries_captured, reset_emails_in_zulip_realm from zerver.lib.timestamp import datetime_to_timestamp from zerver.models import ( Client, PushDeviceToken, UserActivity, UserActivityInterval, - UserProfile, UserPresence, + UserProfile, flush_per_request_caches, ) -import datetime class ActivityTest(ZulipTestCase): @mock.patch("stripe.Customer.list", return_value=[]) diff --git a/zerver/tests/test_push_notifications.py b/zerver/tests/test_push_notifications.py index b2097566fc130..6255fbc1db1d5 100644 --- a/zerver/tests/test_push_notifications.py +++ b/zerver/tests/test_push_notifications.py @@ -1,52 +1,35 @@ -from contextlib import contextmanager +import base64 import datetime import itertools -import requests +import os +import uuid +from contextlib import contextmanager +from typing import Any, Dict, Iterator, List, Optional from unittest import mock from unittest.mock import call -from typing import Any, Dict, Iterator, List, Optional -import base64 -import os +import requests import ujson -import uuid - -from django.test import override_settings from django.conf import settings -from django.http import HttpResponse from django.db import transaction from django.db.models import F +from django.http import HttpResponse +from django.test import override_settings from django.utils.crypto import get_random_string +from django.utils.timezone import now from analytics.lib.counts import CountStat, LoggingCountStat from analytics.models import InstallationCount, RealmCount -from zerver.models import ( - PushDeviceToken, - Message, - UserMessage, - receives_offline_email_notifications, - receives_offline_push_notifications, - receives_online_notifications, - receives_stream_notifications, - get_client, - get_realm, - get_stream, - Recipient, - RealmAuditLog, - Stream, - Subscription, -) from zerver.lib.actions import ( do_delete_messages, do_mark_stream_messages_as_read, do_regenerate_api_key, ) -from zerver.lib.soft_deactivation import do_soft_deactivate_users from zerver.lib.push_notifications import ( + DeviceToken, absolute_avatar_url, b64_to_hex, datetime_to_timestamp, - DeviceToken, get_apns_client, get_display_recipient, get_message_payload_apns, @@ -63,16 +46,38 @@ send_notifications_to_bouncer, send_to_push_bouncer, ) -from zerver.lib.remote_server import send_analytics_to_remote_server, \ - build_analytics_data, PushNotificationBouncerException, PushNotificationBouncerRetryLaterError +from zerver.lib.remote_server import ( + PushNotificationBouncerException, + PushNotificationBouncerRetryLaterError, + build_analytics_data, + send_analytics_to_remote_server, +) from zerver.lib.request import JsonableError -from zerver.lib.test_classes import ( - TestCase, ZulipTestCase, +from zerver.lib.soft_deactivation import do_soft_deactivate_users +from zerver.lib.test_classes import TestCase, ZulipTestCase +from zerver.models import ( + Message, + PushDeviceToken, + RealmAuditLog, + Recipient, + Stream, + Subscription, + UserMessage, + get_client, + get_realm, + get_stream, + receives_offline_email_notifications, + receives_offline_push_notifications, + receives_online_notifications, + receives_stream_notifications, +) +from zilencer.models import ( + RemoteInstallationCount, + RemotePushDeviceToken, + RemoteRealmAuditLog, + RemoteRealmCount, + RemoteZulipServer, ) - -from zilencer.models import RemoteZulipServer, RemotePushDeviceToken, \ - RemoteRealmCount, RemoteInstallationCount, RemoteRealmAuditLog -from django.utils.timezone import now ZERVER_DIR = os.path.dirname(os.path.dirname(__file__)) @@ -1510,6 +1515,7 @@ def test_400_error(self, mock_request: mock.MagicMock) -> None: def test_400_error_invalid_server_key(self) -> None: from zerver.decorator import InvalidZulipServerError + # This is the exception our decorator uses for an invalid Zulip server error_obj = InvalidZulipServerError("testRole") with mock.patch('requests.request', diff --git a/zerver/tests/test_queue.py b/zerver/tests/test_queue.py index 48f36d0f7fa0e..d08b43e172cc8 100644 --- a/zerver/tests/test_queue.py +++ b/zerver/tests/test_queue.py @@ -1,13 +1,13 @@ -from unittest import mock from typing import Any, Dict +from unittest import mock from django.test import override_settings -from pika.exceptions import ConnectionClosed, AMQPConnectionError +from pika.exceptions import AMQPConnectionError, ConnectionClosed -from zerver.lib.queue import TornadoQueueClient, queue_json_publish, \ - get_queue_client +from zerver.lib.queue import TornadoQueueClient, get_queue_client, queue_json_publish from zerver.lib.test_classes import ZulipTestCase + class TestTornadoQueueClient(ZulipTestCase): @mock.patch('zerver.lib.queue.logging.getLogger', autospec=True) @mock.patch('zerver.lib.queue.ExceptionFreeTornadoConnection', autospec=True) diff --git a/zerver/tests/test_queue_worker.py b/zerver/tests/test_queue_worker.py index 2bde78a4c6737..4bfff4fdf5bb4 100644 --- a/zerver/tests/test_queue_worker.py +++ b/zerver/tests/test_queue_worker.py @@ -1,12 +1,12 @@ import os -import time -import ujson import smtplib +import time +from typing import Any, Callable, Dict, List, Mapping, Tuple +from unittest.mock import MagicMock, patch +import ujson from django.conf import settings from django.test import override_settings -from unittest.mock import patch, MagicMock -from typing import Any, Callable, Dict, List, Mapping, Tuple from zerver.lib.email_mirror import RateLimitedRealmMirror from zerver.lib.email_mirror_helpers import encode_email_address @@ -14,18 +14,17 @@ from zerver.lib.rate_limiter import RateLimiterLockingException from zerver.lib.remote_server import PushNotificationBouncerRetryLaterError from zerver.lib.send_email import FromAddress -from zerver.lib.test_helpers import simulated_queue_client from zerver.lib.test_classes import ZulipTestCase -from zerver.models import get_client, UserActivity, PreregistrationUser, \ - get_stream, get_realm +from zerver.lib.test_helpers import simulated_queue_client +from zerver.models import PreregistrationUser, UserActivity, get_client, get_realm, get_stream from zerver.tornado.event_queue import build_offline_notification from zerver.worker import queue_processors from zerver.worker.queue_processors import ( - get_active_worker_queues, - QueueProcessingWorker, EmailSendingWorker, LoopQueueProcessingWorker, MissedMessageWorker, + QueueProcessingWorker, + get_active_worker_queues, ) Event = Dict[str, Any] diff --git a/zerver/tests/test_rate_limiter.py b/zerver/tests/test_rate_limiter.py index 7afa2379f479e..898af5fb37b34 100644 --- a/zerver/tests/test_rate_limiter.py +++ b/zerver/tests/test_rate_limiter.py @@ -1,21 +1,19 @@ +import time +from typing import Dict, List, Tuple, Type +from unittest import mock + from zerver.lib.rate_limiter import ( - add_ratelimit_rule, - remove_ratelimit_rule, RateLimitedObject, RateLimitedUser, RateLimiterBackend, RedisRateLimiterBackend, TornadoInMemoryRateLimiterBackend, + add_ratelimit_rule, + remove_ratelimit_rule, ) - from zerver.lib.test_classes import ZulipTestCase from zerver.lib.utils import generate_random_token -from typing import Dict, List, Tuple, Type - -from unittest import mock -import time - RANDOM_KEY_PREFIX = generate_random_token(32) class RateLimitedTestObject(RateLimitedObject): diff --git a/zerver/tests/test_reactions.py b/zerver/tests/test_reactions.py index 9e1ad910eead3..ff349a3fccfe3 100644 --- a/zerver/tests/test_reactions.py +++ b/zerver/tests/test_reactions.py @@ -1,15 +1,17 @@ -import ujson -from django.http import HttpResponse from typing import Any, Dict, List, Mapping from unittest import mock -from zerver.lib.cache import to_dict_cache_key_id, cache_get +import ujson +from django.http import HttpResponse + +from zerver.lib.cache import cache_get, to_dict_cache_key_id from zerver.lib.emoji import emoji_name_to_emoji_code from zerver.lib.message import extract_message_dict from zerver.lib.request import JsonableError -from zerver.lib.test_helpers import tornado_redirected_to_list from zerver.lib.test_classes import ZulipTestCase -from zerver.models import get_realm, Message, Reaction, RealmEmoji, UserMessage +from zerver.lib.test_helpers import tornado_redirected_to_list +from zerver.models import Message, Reaction, RealmEmoji, UserMessage, get_realm + class ReactionEmojiTest(ZulipTestCase): def test_missing_emoji(self) -> None: diff --git a/zerver/tests/test_realm.py b/zerver/tests/test_realm.py index 89a514ccebf66..e571281bdee49 100644 --- a/zerver/tests/test_realm.py +++ b/zerver/tests/test_realm.py @@ -1,36 +1,43 @@ import datetime -import ujson import re -from unittest import mock from email.utils import parseaddr +from typing import Any, Dict, List, Mapping +from unittest import mock +import ujson from django.conf import settings -from typing import Any, Dict, List, Mapping +from confirmation.models import Confirmation, create_confirmation_link from zerver.lib.actions import ( + do_change_plan_type, do_change_realm_subdomain, - do_set_realm_property, + do_create_realm, do_deactivate_realm, do_deactivate_stream, - do_create_realm, do_scrub_realm, - do_change_plan_type, do_send_realm_reactivation_email, + do_set_realm_property, ) - -from confirmation.models import create_confirmation_link, Confirmation from zerver.lib.realm_description import get_realm_rendered_description, get_realm_text_description from zerver.lib.send_email import send_future_email from zerver.lib.streams import create_stream_if_needed from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.test_helpers import ( - reset_emails_in_zulip_realm, - tornado_redirected_to_list, -) +from zerver.lib.test_helpers import reset_emails_in_zulip_realm, tornado_redirected_to_list from zerver.lib.test_runner import slow -from zerver.models import get_realm, Realm, UserProfile, ScheduledEmail, get_stream, \ - CustomProfileField, Message, UserMessage, Attachment, get_user_profile_by_email, \ - get_user_profile_by_id +from zerver.models import ( + Attachment, + CustomProfileField, + Message, + Realm, + ScheduledEmail, + UserMessage, + UserProfile, + get_realm, + get_stream, + get_user_profile_by_email, + get_user_profile_by_id, +) + class RealmTest(ZulipTestCase): def assert_user_profile_cache_gets_new_name(self, user_profile: UserProfile, diff --git a/zerver/tests/test_realm_domains.py b/zerver/tests/test_realm_domains.py index 872fd78f9d130..65a64c8d7ea52 100644 --- a/zerver/tests/test_realm_domains.py +++ b/zerver/tests/test_realm_domains.py @@ -1,16 +1,18 @@ +import ujson from django.core.exceptions import ValidationError from django.db.utils import IntegrityError -from zerver.lib.actions import do_change_user_role, \ - do_change_realm_domain, do_create_realm, \ - do_remove_realm_domain, do_set_realm_property -from zerver.lib.email_validation import email_allowed_for_realm +from zerver.lib.actions import ( + do_change_realm_domain, + do_change_user_role, + do_create_realm, + do_remove_realm_domain, + do_set_realm_property, +) from zerver.lib.domains import validate_domain +from zerver.lib.email_validation import email_allowed_for_realm from zerver.lib.test_classes import ZulipTestCase -from zerver.models import get_realm, UserProfile, \ - RealmDomain, DomainNotAllowedForRealmError - -import ujson +from zerver.models import DomainNotAllowedForRealmError, RealmDomain, UserProfile, get_realm class RealmDomainTest(ZulipTestCase): diff --git a/zerver/tests/test_realm_emoji.py b/zerver/tests/test_realm_emoji.py index 0a672339c7470..4874fb1fa9ce2 100644 --- a/zerver/tests/test_realm_emoji.py +++ b/zerver/tests/test_realm_emoji.py @@ -1,11 +1,11 @@ from unittest import mock -from zerver.lib.actions import do_create_realm, do_create_user, \ - check_add_realm_emoji +from zerver.lib.actions import check_add_realm_emoji, do_create_realm, do_create_user from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import get_test_image_file from zerver.models import Realm, RealmEmoji, UserProfile, get_realm + class RealmEmojiTest(ZulipTestCase): def create_test_emoji(self, name: str, author: UserProfile) -> RealmEmoji: diff --git a/zerver/tests/test_realm_export.py b/zerver/tests/test_realm_export.py index 0c0a0406991c4..e9af3781dd75e 100644 --- a/zerver/tests/test_realm_export.py +++ b/zerver/tests/test_realm_export.py @@ -1,21 +1,23 @@ +import os from unittest.mock import patch -from analytics.models import RealmCount - -from django.utils.timezone import now as timezone_now +import botocore.exceptions +import ujson from django.conf import settings +from django.utils.timezone import now as timezone_now -from zerver.lib.test_classes import ZulipTestCase +from analytics.models import RealmCount from zerver.lib.exceptions import JsonableError -from zerver.lib.test_helpers import use_s3_backend, create_s3_buckets, \ - create_dummy_file, stdout_suppressed - +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import ( + create_dummy_file, + create_s3_buckets, + stdout_suppressed, + use_s3_backend, +) from zerver.models import RealmAuditLog from zerver.views.realm_export import export_realm -import os -import ujson -import botocore.exceptions class RealmExportTest(ZulipTestCase): """ diff --git a/zerver/tests/test_realm_filters.py b/zerver/tests/test_realm_filters.py index f94c2b18817fc..7b93ece2c599d 100644 --- a/zerver/tests/test_realm_filters.py +++ b/zerver/tests/test_realm_filters.py @@ -4,6 +4,7 @@ from zerver.lib.test_classes import ZulipTestCase from zerver.models import RealmFilter, get_realm + class RealmFilterTest(ZulipTestCase): def test_list(self) -> None: diff --git a/zerver/tests/test_redis_utils.py b/zerver/tests/test_redis_utils.py index db15268f61469..c69a4de40cc76 100644 --- a/zerver/tests/test_redis_utils.py +++ b/zerver/tests/test_redis_utils.py @@ -1,8 +1,15 @@ from unittest import mock +from zerver.lib.redis_utils import ( + MAX_KEY_LENGTH, + ZulipRedisKeyOfWrongFormatError, + ZulipRedisKeyTooLongError, + get_dict_from_redis, + get_redis_client, + put_dict_in_redis, +) from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.redis_utils import get_redis_client, get_dict_from_redis, put_dict_in_redis, \ - ZulipRedisKeyTooLongError, ZulipRedisKeyOfWrongFormatError, MAX_KEY_LENGTH + class RedisUtilsTest(ZulipTestCase): key_format = "test_redis_utils_{token}" diff --git a/zerver/tests/test_report.py b/zerver/tests/test_report.py index 714e70adfe7c4..46d3b4f680e4d 100644 --- a/zerver/tests/test_report.py +++ b/zerver/tests/test_report.py @@ -1,13 +1,12 @@ from typing import Any, Callable, Dict, Iterable, List, Tuple +from unittest import mock +import ujson from django.test import override_settings -from zerver.lib.test_classes import ( - ZulipTestCase, -) + +from zerver.lib.test_classes import ZulipTestCase from zerver.lib.utils import statsd -from unittest import mock -import ujson def fix_params(raw_params: Dict[str, Any]) -> Dict[str, str]: # A few of our few legacy endpoints need their diff --git a/zerver/tests/test_retention.py b/zerver/tests/test_retention.py index 056909aec8f9c..192ab15455546 100644 --- a/zerver/tests/test_retention.py +++ b/zerver/tests/test_retention.py @@ -5,25 +5,39 @@ from django.conf import settings from django.utils.timezone import now as timezone_now -from zerver.lib.actions import internal_send_private_message, do_add_submessage, do_delete_messages -from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.test_helpers import queries_captured -from zerver.lib.upload import create_attachment -from zerver.models import (Message, Realm, Stream, ArchivedUserMessage, SubMessage, - ArchivedMessage, Attachment, ArchivedAttachment, UserMessage, - Reaction, ArchivedReaction, ArchivedSubMessage, ArchiveTransaction, - get_realm, get_user_profile_by_email, get_stream, get_system_bot) +from zerver.lib.actions import do_add_submessage, do_delete_messages, internal_send_private_message from zerver.lib.retention import ( archive_messages, - move_messages_to_archive, - restore_all_data_from_archive, clean_archived_data, get_realms_and_streams_for_archiving, + move_messages_to_archive, + restore_all_data_from_archive, +) +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import queries_captured +from zerver.lib.upload import create_attachment +from zerver.models import ( + ArchivedAttachment, + ArchivedMessage, + ArchivedReaction, + ArchivedSubMessage, + ArchivedUserMessage, + ArchiveTransaction, + Attachment, + Message, + Reaction, + Realm, + Stream, + SubMessage, + UserMessage, + get_realm, + get_stream, + get_system_bot, + get_user_profile_by_email, ) -from zerver.tornado.event_queue import send_event - # Class with helper functions useful for testing archiving of reactions: from zerver.tests.test_reactions import EmojiReactionBase +from zerver.tornado.event_queue import send_event ZULIP_REALM_DAYS = 30 MIT_REALM_DAYS = 100 diff --git a/zerver/tests/test_service_bot_system.py b/zerver/tests/test_service_bot_system.py index 8867ef75d2bd5..69786402460a1 100644 --- a/zerver/tests/test_service_bot_system.py +++ b/zerver/tests/test_service_bot_system.py @@ -1,25 +1,16 @@ +from typing import Any, Callable, Mapping, Union from unittest import mock -from typing import Any, Union, Mapping, Callable +import ujson from django.conf import settings from django.test import override_settings -from zerver.lib.actions import ( - do_create_user, - get_service_bot_events, -) -from zerver.lib.bot_lib import StateHandler, EmbeddedBotHandler, \ - EmbeddedBotEmptyRecipientsList +from zerver.lib.actions import do_create_user, get_service_bot_events +from zerver.lib.bot_config import ConfigError, load_bot_config_template, set_bot_config +from zerver.lib.bot_lib import EmbeddedBotEmptyRecipientsList, EmbeddedBotHandler, StateHandler from zerver.lib.bot_storage import StateError -from zerver.lib.bot_config import set_bot_config, ConfigError, load_bot_config_template from zerver.lib.test_classes import ZulipTestCase -from zerver.models import ( - get_realm, - UserProfile, - Recipient, -) - -import ujson +from zerver.models import Recipient, UserProfile, get_realm BOT_TYPE_TO_QUEUE_NAME = { UserProfile.OUTGOING_WEBHOOK_BOT: 'outgoing_webhooks', diff --git a/zerver/tests/test_sessions.py b/zerver/tests/test_sessions.py index 9b4daf3294983..5b096201a4ffd 100644 --- a/zerver/tests/test_sessions.py +++ b/zerver/tests/test_sessions.py @@ -1,25 +1,22 @@ from datetime import timedelta -from django.utils.timezone import now as timezone_now from typing import Any, Callable +from unittest import mock + +from django.utils.timezone import now as timezone_now from zerver.lib.sessions import ( - user_sessions, + delete_all_deactivated_user_sessions, + delete_all_user_sessions, + delete_realm_user_sessions, delete_session, delete_user_sessions, - delete_realm_user_sessions, - delete_all_user_sessions, - delete_all_deactivated_user_sessions, get_expirable_session_var, set_expirable_session_var, + user_sessions, ) - -from zerver.models import ( - get_realm, Realm, UserProfile, -) - from zerver.lib.test_classes import ZulipTestCase +from zerver.models import Realm, UserProfile, get_realm -from unittest import mock class TestSessions(ZulipTestCase): diff --git a/zerver/tests/test_settings.py b/zerver/tests/test_settings.py index 4b70aa01d0fc8..246ae6f687a61 100644 --- a/zerver/tests/test_settings.py +++ b/zerver/tests/test_settings.py @@ -1,20 +1,18 @@ -from unittest import mock import time -import ujson +from typing import Any, Dict +from unittest import mock +import ujson from django.http import HttpResponse from django.test import override_settings -from typing import Any, Dict from zerver.lib.initial_password import initial_password +from zerver.lib.rate_limiter import add_ratelimit_rule, remove_ratelimit_rule from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import get_test_image_file from zerver.lib.users import get_all_api_keys -from zerver.lib.rate_limiter import add_ratelimit_rule, remove_ratelimit_rule -from zerver.models import ( - UserProfile, - get_user_profile_by_api_key, -) +from zerver.models import UserProfile, get_user_profile_by_api_key + class ChangeSettingsTest(ZulipTestCase): diff --git a/zerver/tests/test_signup.py b/zerver/tests/test_signup.py index 242e0078625fd..9bc375b8acdd4 100644 --- a/zerver/tests/test_signup.py +++ b/zerver/tests/test_signup.py @@ -1,86 +1,102 @@ import datetime +import re +import smtplib +import time +import urllib from email.utils import parseaddr +from typing import Any, List, Optional +from unittest.mock import MagicMock, patch +import ujson from django.conf import settings from django.contrib.auth.views import INTERNAL_RESET_URL_TOKEN from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ValidationError from django.http import HttpResponse from django.test import TestCase, override_settings -from django.utils.timezone import now as timezone_now -from django.core.exceptions import ValidationError from django.urls import reverse +from django.utils.timezone import now as timezone_now -from unittest.mock import patch, MagicMock -from zerver.lib.test_helpers import ( - avatar_disk_path, - get_test_image_file, - reset_emails_in_zulip_realm, -) - -from confirmation.models import Confirmation, create_confirmation_link, MultiuseInvite, \ - generate_key, confirmation_url, get_object_from_key, ConfirmationKeyException, \ - one_click_unsubscribe_link from confirmation import settings as confirmation_settings - -from zerver.forms import HomepageForm, check_subdomain_available -from zerver.decorator import do_two_factor_login -from zerver.views.auth import \ - redirect_and_log_into_subdomain, start_two_factor_auth -from zerver.views.invite import get_invitee_emails_set -from zerver.views.development.registration import confirmation_key - -from zerver.models import ( - get_realm, get_user, CustomProfileField, - CustomProfileFieldValue, DefaultStream, PreregistrationUser, - Realm, Recipient, Message, ScheduledEmail, UserProfile, UserMessage, - Stream, Subscription, flush_per_request_caches, get_system_bot, - get_user_by_delivery_email, +from confirmation.models import ( + Confirmation, + ConfirmationKeyException, + MultiuseInvite, + confirmation_url, + create_confirmation_link, + generate_key, + get_object_from_key, + one_click_unsubscribe_link, ) +from zerver.context_processors import common_context +from zerver.decorator import do_two_factor_login +from zerver.forms import HomepageForm, check_subdomain_available from zerver.lib.actions import ( + add_new_user_history, + do_add_default_stream, + do_change_full_name, do_change_user_role, - get_stream, do_create_default_stream_group, - do_add_default_stream, do_create_realm, - get_default_streams_for_realm, - do_invite_users, do_create_user) -from zerver.lib.send_email import send_future_email, FromAddress, \ - deliver_email -from zerver.lib.initial_password import initial_password -from zerver.lib.actions import ( - do_get_user_invites, - do_change_full_name, + do_create_user, do_deactivate_realm, do_deactivate_user, + do_get_user_invites, + do_invite_users, do_set_realm_property, - add_new_user_history, + get_default_streams_for_realm, + get_stream, +) +from zerver.lib.email_notifications import enqueue_welcome_emails, followup_day2_email_delay +from zerver.lib.initial_password import initial_password +from zerver.lib.mobile_auth_otp import ( + ascii_to_hex, + hex_to_ascii, + is_valid_otp, + otp_decrypt_api_key, + otp_encrypt_api_key, + xor_hex_strings, ) -from zerver.lib.mobile_auth_otp import xor_hex_strings, ascii_to_hex, \ - otp_encrypt_api_key, is_valid_otp, hex_to_ascii, otp_decrypt_api_key -from zerver.lib.email_notifications import enqueue_welcome_emails, \ - followup_day2_email_delay +from zerver.lib.name_restrictions import is_disposable_domain from zerver.lib.rate_limiter import add_ratelimit_rule, remove_ratelimit_rule -from zerver.lib.subdomains import is_root_domain_available +from zerver.lib.send_email import FromAddress, deliver_email, send_future_email from zerver.lib.stream_subscription import get_stream_subscriptions_for_user from zerver.lib.streams import create_stream_if_needed -from zerver.lib.test_helpers import find_key_by_email, queries_captured, \ - HostRequestMock, load_subdomain_token -from zerver.lib.test_classes import ( - ZulipTestCase, +from zerver.lib.subdomains import is_root_domain_available +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import ( + HostRequestMock, + avatar_disk_path, + find_key_by_email, + get_test_image_file, + load_subdomain_token, + queries_captured, + reset_emails_in_zulip_realm, ) -from zerver.lib.name_restrictions import is_disposable_domain -from zerver.context_processors import common_context - -from zproject.backends import ExternalAuthResult, ExternalAuthDataDict - -import re -import smtplib -import time -import ujson - -from typing import Any, List, Optional +from zerver.models import ( + CustomProfileField, + CustomProfileFieldValue, + DefaultStream, + Message, + PreregistrationUser, + Realm, + Recipient, + ScheduledEmail, + Stream, + Subscription, + UserMessage, + UserProfile, + flush_per_request_caches, + get_realm, + get_system_bot, + get_user, + get_user_by_delivery_email, +) +from zerver.views.auth import redirect_and_log_into_subdomain, start_two_factor_auth +from zerver.views.development.registration import confirmation_key +from zerver.views.invite import get_invitee_emails_set +from zproject.backends import ExternalAuthDataDict, ExternalAuthResult -import urllib class RedirectAndLogIntoSubdomainTestCase(ZulipTestCase): def test_data(self) -> None: @@ -3968,6 +3984,7 @@ def test_result(self) -> None: self.assertIn(self.example_email("iago"), content) self.assertIn(self.example_email("cordelia"), content) from django.core.mail import outbox + # 3 = 1 + 2 -- Cordelia gets an email each for the "zulip" and "lear" realms. self.assertEqual(len(outbox), 3) diff --git a/zerver/tests/test_slack_importer.py b/zerver/tests/test_slack_importer.py index 3d9b5df5456c2..3e0ec99122024 100644 --- a/zerver/tests/test_slack_importer.py +++ b/zerver/tests/test_slack_importer.py @@ -1,64 +1,49 @@ +import logging +import os +import shutil +from typing import Any, Dict, Iterator, List, Set, Tuple +from unittest import mock +from unittest.mock import ANY, call + +import ujson from django.conf import settings from django.utils.timezone import now as timezone_now +from zerver.data_import.import_util import ( + build_defaultstream, + build_recipient, + build_subscription, + build_usermessages, + build_zerver_realm, +) +from zerver.data_import.sequencer import NEXT_ID from zerver.data_import.slack import ( - get_slack_api_data, - get_owner, - get_admin, - get_guest, - get_user_timezone, - fetch_shared_channel_users, - users_to_zerver_userprofile, - get_subscription, - channels_to_zerver_stream, - slack_workspace_to_realm, - get_message_sending_user, - channel_message_to_zerver_message, - convert_slack_workspace_messages, - do_convert_data, - process_message_files, AddedChannelsT, AddedMPIMsT, DMMembersT, ZerverFieldsT, + channel_message_to_zerver_message, + channels_to_zerver_stream, + convert_slack_workspace_messages, + do_convert_data, + fetch_shared_channel_users, + get_admin, + get_guest, + get_message_sending_user, + get_owner, + get_slack_api_data, + get_subscription, + get_user_timezone, + process_message_files, + slack_workspace_to_realm, + users_to_zerver_userprofile, ) -from zerver.data_import.import_util import ( - build_zerver_realm, - build_subscription, - build_recipient, - build_usermessages, - build_defaultstream, -) -from zerver.data_import.sequencer import ( - NEXT_ID, -) -from zerver.lib.import_realm import ( - do_import_realm, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.lib.test_helpers import ( - get_test_image_file, -) -from zerver.lib.topic import ( - EXPORT_TOPIC_NAME, -) -from zerver.models import ( - Realm, - get_realm, - RealmAuditLog, - Recipient, - UserProfile, -) +from zerver.lib.import_realm import do_import_realm +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import get_test_image_file +from zerver.lib.topic import EXPORT_TOPIC_NAME +from zerver.models import Realm, RealmAuditLog, Recipient, UserProfile, get_realm -import ujson -import logging -import shutil -import os -from unittest import mock -from unittest.mock import ANY, call -from typing import Any, Dict, List, Set, Tuple, Iterator def remove_folder(path: str) -> None: if os.path.exists(path): diff --git a/zerver/tests/test_slack_message_conversion.py b/zerver/tests/test_slack_message_conversion.py index 11c6154590135..2fc2063e0a746 100644 --- a/zerver/tests/test_slack_message_conversion.py +++ b/zerver/tests/test_slack_message_conversion.py @@ -1,16 +1,16 @@ +import os +from typing import Any, Dict, List, Tuple + +import ujson + from zerver.data_import.slack_message_conversion import ( convert_to_zulip_markdown, get_user_full_name, ) -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.lib.test_runner import slow from zerver.lib import mdiff -import ujson +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_runner import slow -import os -from typing import Any, Dict, List, Tuple class SlackMessageConversion(ZulipTestCase): def assertEqual(self, first: Any, second: Any, msg: str="") -> None: diff --git a/zerver/tests/test_soft_deactivation.py b/zerver/tests/test_soft_deactivation.py index c7eda6dba81f0..45fb69fb792a4 100644 --- a/zerver/tests/test_soft_deactivation.py +++ b/zerver/tests/test_soft_deactivation.py @@ -3,18 +3,17 @@ from django.utils.timezone import now as timezone_now from zerver.lib.soft_deactivation import ( + do_auto_soft_deactivate_users, + do_catch_up_soft_deactivated_users, + do_soft_activate_users, do_soft_deactivate_user, do_soft_deactivate_users, - get_users_for_soft_deactivation, - do_soft_activate_users, get_soft_deactivated_users_for_catch_up, - do_catch_up_soft_deactivated_users, - do_auto_soft_deactivate_users, + get_users_for_soft_deactivation, ) from zerver.lib.test_classes import ZulipTestCase -from zerver.models import ( - Client, UserProfile, UserActivity, get_realm, UserMessage, -) +from zerver.models import Client, UserActivity, UserMessage, UserProfile, get_realm + class UserSoftDeactivationTests(ZulipTestCase): diff --git a/zerver/tests/test_subdomains.py b/zerver/tests/test_subdomains.py index 2b6a37f875a86..25ad1edd904cd 100644 --- a/zerver/tests/test_subdomains.py +++ b/zerver/tests/test_subdomains.py @@ -1,11 +1,12 @@ -from unittest import mock from typing import Any, Dict, List +from unittest import mock from django.test import TestCase from zerver.lib.subdomains import get_subdomain from zerver.models import Realm + class SubdomainsTest(TestCase): def test_get_subdomain(self) -> None: diff --git a/zerver/tests/test_submessage.py b/zerver/tests/test_submessage.py index e3785c599cf54..e58620005b860 100644 --- a/zerver/tests/test_submessage.py +++ b/zerver/tests/test_submessage.py @@ -1,17 +1,10 @@ +from typing import Any, Dict, List from unittest import mock +from zerver.lib.message import MessageDict from zerver.lib.test_classes import ZulipTestCase +from zerver.models import Message, SubMessage -from zerver.lib.message import ( - MessageDict, -) - -from zerver.models import ( - Message, - SubMessage, -) - -from typing import Any, Dict, List class TestBasics(ZulipTestCase): def test_get_raw_db_rows(self) -> None: diff --git a/zerver/tests/test_subs.py b/zerver/tests/test_subs.py index 6257055dcb40d..ffcfda63fecee 100644 --- a/zerver/tests/test_subs.py +++ b/zerver/tests/test_subs.py @@ -1,87 +1,89 @@ +import random +from datetime import timedelta from typing import Any, Dict, List, Mapping, Optional, Set, Union +from unittest import mock +import ujson from django.conf import settings from django.core.exceptions import ValidationError from django.http import HttpRequest, HttpResponse from django.utils.timezone import now as timezone_now +from zerver.decorator import JsonableError from zerver.lib import cache - -from zerver.lib.test_helpers import ( - get_subscription, queries_captured, tornado_redirected_to_list, - reset_emails_in_zulip_realm, -) - -from zerver.lib.test_classes import ( - ZulipTestCase, -) - -from zerver.decorator import ( - JsonableError, -) - -from zerver.lib.response import ( - json_error, - json_success, -) - -from zerver.lib.streams import ( - access_stream_by_id, access_stream_by_name, filter_stream_authorization, - list_to_streams, create_streams_if_needed, -) - -from zerver.lib.stream_subscription import ( - get_active_subscriptions_for_stream_id, - num_subscribers_for_stream_id, -) - -from zerver.lib.test_runner import ( - slow, -) - -from zerver.models import ( - Realm, Recipient, Stream, Subscription, - DefaultStream, UserProfile, active_non_guest_user_ids, - get_default_stream_groups, flush_per_request_caches, DefaultStreamGroup, - get_client, get_realm, get_user, get_user_profile_by_id_in_realm, Message, UserMessage, -) - from zerver.lib.actions import ( - do_add_default_stream, do_change_user_role, do_set_realm_property, - do_create_realm, do_remove_default_stream, bulk_get_subscriber_user_ids, - gather_subscriptions_helper, bulk_add_subscriptions, bulk_remove_subscriptions, - gather_subscriptions, get_default_streams_for_realm, get_stream, - do_get_streams, + bulk_add_subscriptions, + bulk_get_subscriber_user_ids, + bulk_remove_subscriptions, + can_access_stream_user_ids, create_stream_if_needed, - ensure_stream, + do_add_default_stream, + do_add_streams_to_default_stream_group, + do_change_default_stream_group_description, + do_change_default_stream_group_name, + do_change_stream_post_policy, + do_change_user_role, + do_create_default_stream_group, + do_create_realm, do_deactivate_stream, do_deactivate_user, - do_create_default_stream_group, - do_add_streams_to_default_stream_group, do_remove_streams_from_default_stream_group, + do_get_streams, + do_remove_default_stream, do_remove_default_stream_group, - do_change_default_stream_group_description, - do_change_default_stream_group_name, + do_remove_streams_from_default_stream_group, + do_set_realm_property, + ensure_stream, + gather_subscriptions, + gather_subscriptions_helper, + get_average_weekly_stream_traffic, + get_default_streams_for_realm, + get_stream, lookup_default_stream_groups, - can_access_stream_user_ids, + round_to_2_significant_digits, validate_user_access_to_subscribers_helper, - get_average_weekly_stream_traffic, round_to_2_significant_digits, - do_change_stream_post_policy, ) - -from zerver.views.streams import ( - compose_views, +from zerver.lib.message import aggregate_unread_data, get_raw_unread_data +from zerver.lib.response import json_error, json_success +from zerver.lib.stream_recipient import StreamRecipientMap +from zerver.lib.stream_subscription import ( + get_active_subscriptions_for_stream_id, + num_subscribers_for_stream_id, ) - -from zerver.lib.message import ( - aggregate_unread_data, - get_raw_unread_data, +from zerver.lib.streams import ( + access_stream_by_id, + access_stream_by_name, + create_streams_if_needed, + filter_stream_authorization, + list_to_streams, ) -from zerver.lib.stream_recipient import StreamRecipientMap +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import ( + get_subscription, + queries_captured, + reset_emails_in_zulip_realm, + tornado_redirected_to_list, +) +from zerver.lib.test_runner import slow +from zerver.models import ( + DefaultStream, + DefaultStreamGroup, + Message, + Realm, + Recipient, + Stream, + Subscription, + UserMessage, + UserProfile, + active_non_guest_user_ids, + flush_per_request_caches, + get_client, + get_default_stream_groups, + get_realm, + get_user, + get_user_profile_by_id_in_realm, +) +from zerver.views.streams import compose_views -from datetime import timedelta -from unittest import mock -import random -import ujson class TestMiscStuff(ZulipTestCase): def test_empty_results(self) -> None: diff --git a/zerver/tests/test_templates.py b/zerver/tests/test_templates.py index a35d282ab4477..b5cdc86ee8e6d 100644 --- a/zerver/tests/test_templates.py +++ b/zerver/tests/test_templates.py @@ -1,9 +1,7 @@ from django.template.loader import get_template from zerver.lib.exceptions import InvalidMarkdownIncludeStatement -from zerver.lib.test_classes import ( - ZulipTestCase, -) +from zerver.lib.test_classes import ZulipTestCase class TemplateTestCase(ZulipTestCase): diff --git a/zerver/tests/test_thumbnail.py b/zerver/tests/test_thumbnail.py index 6653093beaa0b..d707a28cf44ca 100644 --- a/zerver/tests/test_thumbnail.py +++ b/zerver/tests/test_thumbnail.py @@ -1,19 +1,20 @@ +import base64 +import urllib +from io import StringIO + +import ujson from django.conf import settings from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import ( - use_s3_backend, create_s3_buckets, - override_settings, get_test_image_file, + override_settings, + use_s3_backend, ) from zerver.lib.upload import upload_backend, upload_emoji_image from zerver.lib.users import get_api_key -from io import StringIO -import ujson -import urllib -import base64 class ThumbnailTest(ZulipTestCase): diff --git a/zerver/tests/test_timestamp.py b/zerver/tests/test_timestamp.py index cd2aa739c503c..5460d510b5dc4 100644 --- a/zerver/tests/test_timestamp.py +++ b/zerver/tests/test_timestamp.py @@ -1,11 +1,19 @@ -from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.timestamp import floor_to_hour, floor_to_day, ceiling_to_hour, \ - timestamp_to_datetime, datetime_to_timestamp, \ - TimezoneNotUTCException, convert_to_UTC - from datetime import timedelta, timezone + from dateutil import parser +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.timestamp import ( + TimezoneNotUTCException, + ceiling_to_hour, + convert_to_UTC, + datetime_to_timestamp, + floor_to_day, + floor_to_hour, + timestamp_to_datetime, +) + + class TestTimestamp(ZulipTestCase): def test_datetime_and_timestamp_conversions(self) -> None: timestamp = 1483228800 diff --git a/zerver/tests/test_tornado.py b/zerver/tests/test_tornado.py index 9b377dddf63c8..bdb519e4b202f 100644 --- a/zerver/tests/test_tornado.py +++ b/zerver/tests/test_tornado.py @@ -1,22 +1,20 @@ +import urllib.parse +from typing import Any, Dict, List, Optional, cast + import ujson from django.conf import settings -from django.db import close_old_connections from django.core import signals +from django.db import close_old_connections from django.test import override_settings from tornado.httpclient import HTTPResponse - -from zerver.lib.test_classes import ZulipTestCase - from tornado.testing import AsyncHTTPTestCase from tornado.web import Application -from zerver.tornado.application import create_tornado_application +from zerver.lib.test_classes import ZulipTestCase from zerver.tornado import event_queue +from zerver.tornado.application import create_tornado_application from zerver.tornado.event_queue import process_event -import urllib.parse - -from typing import Any, Dict, Optional, List, cast class TornadoWebTestCase(AsyncHTTPTestCase, ZulipTestCase): def setUp(self) -> None: diff --git a/zerver/tests/test_transfer.py b/zerver/tests/test_transfer.py index ec642b2b634e0..f1274e74d0aed 100644 --- a/zerver/tests/test_transfer.py +++ b/zerver/tests/test_transfer.py @@ -1,18 +1,22 @@ -from django.conf import settings +import logging +from unittest.mock import Mock, patch +from django.conf import settings from moto import mock_s3 -from unittest.mock import Mock, patch -import logging -from zerver.lib.upload import upload_message_file, resize_emoji -from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.test_helpers import create_s3_buckets, get_test_image_file, \ - avatar_disk_path -from zerver.lib.transfer import transfer_avatars_to_s3, transfer_emoji_to_s3, \ - transfer_message_files_to_s3, transfer_uploads_to_s3 -from zerver.models import Attachment, RealmEmoji from zerver.lib.actions import check_add_realm_emoji from zerver.lib.avatar_hash import user_avatar_path +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import avatar_disk_path, create_s3_buckets, get_test_image_file +from zerver.lib.transfer import ( + transfer_avatars_to_s3, + transfer_emoji_to_s3, + transfer_message_files_to_s3, + transfer_uploads_to_s3, +) +from zerver.lib.upload import resize_emoji, upload_message_file +from zerver.models import Attachment, RealmEmoji + class TransferUploadsToS3Test(ZulipTestCase): logger = logging.getLogger() diff --git a/zerver/tests/test_tutorial.py b/zerver/tests/test_tutorial.py index 1ac2f6e06ff23..7332a25ee50c4 100644 --- a/zerver/tests/test_tutorial.py +++ b/zerver/tests/test_tutorial.py @@ -1,11 +1,11 @@ +import ujson from django.conf import settings from zerver.lib.actions import internal_send_private_message from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import message_stream_count, most_recent_message -from zerver.models import get_system_bot, UserProfile +from zerver.models import UserProfile, get_system_bot -import ujson class TutorialTests(ZulipTestCase): def setUp(self) -> None: diff --git a/zerver/tests/test_type_debug.py b/zerver/tests/test_type_debug.py index 7c66fc11e3ce1..9bc9fbd6e218a 100644 --- a/zerver/tests/test_type_debug.py +++ b/zerver/tests/test_type_debug.py @@ -1,11 +1,10 @@ import sys -from unittest import TestCase from io import StringIO +from typing import Any, Callable, Dict, Iterable, List, Tuple, TypeVar +from unittest import TestCase from zerver.lib.type_debug import print_types -from typing import Any, Callable, Dict, Iterable, Tuple, TypeVar, List - T = TypeVar('T') def add(x: Any=0, y: Any=0) -> Any: diff --git a/zerver/tests/test_typing.py b/zerver/tests/test_typing.py index e85ca32e6aa70..750b293b7ab65 100644 --- a/zerver/tests/test_typing.py +++ b/zerver/tests/test_typing.py @@ -1,17 +1,11 @@ +from typing import Any, List, Mapping + import ujson -from typing import Any, Mapping, List - -from zerver.lib.test_helpers import ( - tornado_redirected_to_list, - queries_captured, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.models import ( - Huddle, - get_huddle_hash, -) + +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import queries_captured, tornado_redirected_to_list +from zerver.models import Huddle, get_huddle_hash + class TypingValidateOperatorTest(ZulipTestCase): def test_missing_parameter(self) -> None: diff --git a/zerver/tests/test_unread.py b/zerver/tests/test_unread.py index a427d4988f6ee..803e1cf73932c 100644 --- a/zerver/tests/test_unread.py +++ b/zerver/tests/test_unread.py @@ -1,32 +1,15 @@ from typing import Any, List, Mapping +from unittest import mock +import ujson from django.db import connection -from zerver.models import ( - get_realm, - get_stream, - get_user, - Subscription, - UserMessage, - UserProfile, -) - -from zerver.lib.fix_unreads import ( - fix, - fix_pre_pointer, - fix_unsubscribed, -) -from zerver.lib.test_helpers import ( - get_subscription, - tornado_redirected_to_list, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) +from zerver.lib.fix_unreads import fix, fix_pre_pointer, fix_unsubscribed +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import get_subscription, tornado_redirected_to_list from zerver.lib.topic_mutes import add_topic_mute +from zerver.models import Subscription, UserMessage, UserProfile, get_realm, get_stream, get_user -from unittest import mock -import ujson class PointerTest(ZulipTestCase): diff --git a/zerver/tests/test_upload.py b/zerver/tests/test_upload.py index 1e08e1ceef84e..fa0084658c5b3 100644 --- a/zerver/tests/test_upload.py +++ b/zerver/tests/test_upload.py @@ -1,63 +1,80 @@ +import datetime +import io +import os +import re +import shutil +import time +import urllib +from io import StringIO +from unittest import mock +from unittest.mock import patch + +import botocore.exceptions +import ujson from django.conf import settings from django.test import TestCase -from unittest.mock import patch +from django.utils.timezone import now as timezone_now +from django_sendfile.sendfile import _get_sendfile +from PIL import Image -from zerver.lib.avatar import ( - avatar_url, - get_avatar_field, +import zerver.lib.upload +from scripts.lib.zulip_tools import get_dev_uuid_var_path +from zerver.lib.actions import ( + do_change_icon_source, + do_change_logo_source, + do_change_plan_type, + do_delete_old_unclaimed_attachments, + do_set_realm_property, + internal_send_private_message, ) +from zerver.lib.avatar import avatar_url, get_avatar_field from zerver.lib.avatar_hash import user_avatar_path +from zerver.lib.cache import cache_get, get_realm_used_upload_space_cache_key +from zerver.lib.create_user import copy_user_settings from zerver.lib.initial_password import initial_password from zerver.lib.realm_icon import realm_icon_url from zerver.lib.realm_logo import get_realm_logo_url -from zerver.lib.test_classes import ZulipTestCase, UploadSerializeMixin +from zerver.lib.test_classes import UploadSerializeMixin, ZulipTestCase from zerver.lib.test_helpers import ( avatar_disk_path, - get_test_image_file, - use_s3_backend, create_s3_buckets, + get_test_image_file, queries_captured, + use_s3_backend, ) -from zerver.lib.upload import sanitize_name, S3UploadBackend, \ - upload_message_file, upload_emoji_image, delete_message_image, LocalUploadBackend, \ - ZulipUploadBackend, MEDIUM_AVATAR_SIZE, resize_avatar, \ - resize_emoji, BadImageError, get_realm_for_filename, \ - DEFAULT_AVATAR_SIZE, DEFAULT_EMOJI_SIZE, exif_rotate, \ - upload_export_tarball, delete_export_tarball -import zerver.lib.upload -from zerver.models import Attachment, get_user_by_delivery_email, \ - Message, UserProfile, Realm, \ - RealmDomain, RealmEmoji, get_realm, get_system_bot, \ - validate_attachment_request -from zerver.lib.actions import ( - do_change_plan_type, - do_change_icon_source, - do_change_logo_source, - do_delete_old_unclaimed_attachments, - internal_send_private_message, - do_set_realm_property, +from zerver.lib.upload import ( + DEFAULT_AVATAR_SIZE, + DEFAULT_EMOJI_SIZE, + MEDIUM_AVATAR_SIZE, + BadImageError, + LocalUploadBackend, + S3UploadBackend, + ZulipUploadBackend, + delete_export_tarball, + delete_message_image, + exif_rotate, + get_realm_for_filename, + resize_avatar, + resize_emoji, + sanitize_name, + upload_emoji_image, + upload_export_tarball, + upload_message_file, ) -from zerver.lib.cache import get_realm_used_upload_space_cache_key, cache_get -from zerver.lib.create_user import copy_user_settings from zerver.lib.users import get_api_key +from zerver.models import ( + Attachment, + Message, + Realm, + RealmDomain, + RealmEmoji, + UserProfile, + get_realm, + get_system_bot, + get_user_by_delivery_email, + validate_attachment_request, +) -from scripts.lib.zulip_tools import get_dev_uuid_var_path - -import urllib -import ujson -from PIL import Image -import botocore.exceptions - -from io import StringIO -from unittest import mock -import os -import io -import shutil -import re -import time -import datetime -from django.utils.timezone import now as timezone_now -from django_sendfile.sendfile import _get_sendfile def destroy_uploads() -> None: if os.path.exists(settings.LOCAL_UPLOADS_DIR): diff --git a/zerver/tests/test_urls.py b/zerver/tests/test_urls.py index 60b4555fa02c8..10a2de74f926b 100644 --- a/zerver/tests/test_urls.py +++ b/zerver/tests/test_urls.py @@ -1,16 +1,17 @@ import importlib import os -import ujson +from typing import List, Optional import django.urls.resolvers -from django.test import TestCase, Client -from typing import List, Optional +import ujson +from django.test import Client, TestCase from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_runner import slow from zerver.models import Stream from zproject import urls + class PublicURLTest(ZulipTestCase): """ Account creation URLs are accessible even when not logged in. Authenticated diff --git a/zerver/tests/test_user_groups.py b/zerver/tests/test_user_groups.py index 423a533dcb43f..3ac492c780a79 100644 --- a/zerver/tests/test_user_groups.py +++ b/zerver/tests/test_user_groups.py @@ -1,26 +1,21 @@ -import ujson from unittest import mock -from zerver.lib.actions import ( - ensure_stream, -) +import ujson +from zerver.lib.actions import do_set_realm_property, ensure_stream from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.test_helpers import ( - most_recent_usermessage, -) -from zerver.lib.actions import do_set_realm_property +from zerver.lib.test_helpers import most_recent_usermessage from zerver.lib.user_groups import ( check_add_user_to_user_group, check_remove_user_from_user_group, create_user_group, + get_memberships_of_users, get_user_groups, user_groups_in_realm, - get_memberships_of_users, user_groups_in_realm_serialized, ) -from zerver.models import UserGroup, get_realm, Realm, \ - UserGroupMembership +from zerver.models import Realm, UserGroup, UserGroupMembership, get_realm + class UserGroupTestCase(ZulipTestCase): def create_user_group_for_test(self, group_name: str, diff --git a/zerver/tests/test_user_status.py b/zerver/tests/test_user_status.py index 051b7606c1d65..5c178c64f739b 100644 --- a/zerver/tests/test_user_status.py +++ b/zerver/tests/test_user_status.py @@ -1,24 +1,12 @@ +from typing import Any, Dict, Set + import ujson -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.lib.test_helpers import ( - EventInfo, - capture_event, -) -from zerver.lib.user_status import ( - get_user_info_dict, - update_user_status, -) - -from zerver.models import ( - get_client, - UserProfile, - UserStatus, -) +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import EventInfo, capture_event +from zerver.lib.user_status import get_user_info_dict, update_user_status +from zerver.models import UserProfile, UserStatus, get_client -from typing import Any, Dict, Set def get_away_user_ids(realm_id: int) -> Set[int]: user_dict = get_user_info_dict(realm_id) diff --git a/zerver/tests/test_users.py b/zerver/tests/test_users.py index cc4a7278e78a4..cf9213d117dd5 100644 --- a/zerver/tests/test_users.py +++ b/zerver/tests/test_users.py @@ -1,55 +1,59 @@ +import datetime from email.utils import parseaddr +from typing import Any, Dict, Iterable, List, Mapping, Optional, TypeVar, Union +from unittest import mock -from typing import (Any, Dict, Iterable, List, Mapping, - Optional, TypeVar, Union) - -from zerver.lib.test_helpers import ( - get_subscription, - most_recent_message, - queries_captured, - reset_emails_in_zulip_realm, - simulated_empty_cache, - tornado_redirected_to_list, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) - -from zerver.models import UserProfile, Recipient, Realm, \ - RealmDomain, UserHotspot, get_client, \ - get_user, get_user_by_delivery_email, get_realm, get_stream, \ - get_source_profile, get_system_bot, \ - ScheduledEmail, check_valid_user_ids, \ - get_user_by_id_in_realm_including_cross_realm, CustomProfileField, \ - InvalidFakeEmailDomain, get_fake_email_domain +import ujson +from django.conf import settings +from django.test import override_settings -from zerver.lib.avatar import avatar_url, get_gravatar_url -from zerver.lib.exceptions import JsonableError -from zerver.lib.send_email import send_future_email, clear_scheduled_emails, \ - deliver_email from zerver.lib.actions import ( create_users, - get_emails_from_user_ids, - get_recipient_info, - do_deactivate_user, - do_reactivate_user, do_change_user_role, do_create_user, + do_deactivate_user, + do_reactivate_user, do_set_realm_property, + get_emails_from_user_ids, + get_recipient_info, ) +from zerver.lib.avatar import avatar_url, get_gravatar_url from zerver.lib.create_user import copy_user_settings from zerver.lib.events import do_events_register -from zerver.lib.topic_mutes import add_topic_mute +from zerver.lib.exceptions import JsonableError +from zerver.lib.send_email import clear_scheduled_emails, deliver_email, send_future_email from zerver.lib.stream_topic import StreamTopicTarget -from zerver.lib.users import user_ids_to_users, access_user_by_id, \ - get_accounts_for_email - -from django.conf import settings -from django.test import override_settings - -import datetime -from unittest import mock -import ujson +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import ( + get_subscription, + most_recent_message, + queries_captured, + reset_emails_in_zulip_realm, + simulated_empty_cache, + tornado_redirected_to_list, +) +from zerver.lib.topic_mutes import add_topic_mute +from zerver.lib.users import access_user_by_id, get_accounts_for_email, user_ids_to_users +from zerver.models import ( + CustomProfileField, + InvalidFakeEmailDomain, + Realm, + RealmDomain, + Recipient, + ScheduledEmail, + UserHotspot, + UserProfile, + check_valid_user_ids, + get_client, + get_fake_email_domain, + get_realm, + get_source_profile, + get_stream, + get_system_bot, + get_user, + get_user_by_delivery_email, + get_user_by_id_in_realm_including_cross_realm, +) K = TypeVar('K') V = TypeVar('V') diff --git a/zerver/tests/test_webhooks_common.py b/zerver/tests/test_webhooks_common.py index cf4b3ebeebf14..a842b230b1e74 100644 --- a/zerver/tests/test_webhooks_common.py +++ b/zerver/tests/test_webhooks_common.py @@ -1,20 +1,24 @@ from types import SimpleNamespace -from unittest.mock import MagicMock, patch from typing import Dict +from unittest.mock import MagicMock, patch from django.http import HttpRequest from zerver.decorator import api_key_only_webhook_view from zerver.lib.exceptions import InvalidJSONError, JsonableError -from zerver.lib.test_classes import ZulipTestCase, WebhookTestCase -from zerver.lib.webhooks.common import \ - validate_extract_webhook_http_header, \ - MISSING_EVENT_HEADER_MESSAGE, MissingHTTPEventHeader, \ - INVALID_JSON_MESSAGE, get_fixture_http_headers, standardize_headers -from zerver.models import get_user, get_realm, UserProfile -from zerver.lib.users import get_api_key from zerver.lib.send_email import FromAddress +from zerver.lib.test_classes import WebhookTestCase, ZulipTestCase from zerver.lib.test_helpers import HostRequestMock +from zerver.lib.users import get_api_key +from zerver.lib.webhooks.common import ( + INVALID_JSON_MESSAGE, + MISSING_EVENT_HEADER_MESSAGE, + MissingHTTPEventHeader, + get_fixture_http_headers, + standardize_headers, + validate_extract_webhook_http_header, +) +from zerver.models import UserProfile, get_realm, get_user class WebhooksCommonTestCase(ZulipTestCase): diff --git a/zerver/tests/test_widgets.py b/zerver/tests/test_widgets.py index 921827793304d..d362930c3029c 100644 --- a/zerver/tests/test_widgets.py +++ b/zerver/tests/test_widgets.py @@ -1,14 +1,12 @@ -import ujson - -from typing import Dict, Any +from typing import Any, Dict -from zerver.models import SubMessage +import ujson from zerver.lib.test_classes import ZulipTestCase - +from zerver.lib.validator import check_widget_content from zerver.lib.widget import get_widget_data +from zerver.models import SubMessage -from zerver.lib.validator import check_widget_content class WidgetContentTestCase(ZulipTestCase): def test_validation(self) -> None: diff --git a/zerver/tests/test_zcommand.py b/zerver/tests/test_zcommand.py index 31165272e2f0a..08c1ca7d160cd 100644 --- a/zerver/tests/test_zcommand.py +++ b/zerver/tests/test_zcommand.py @@ -1,6 +1,5 @@ -from zerver.lib.test_classes import ( - ZulipTestCase, -) +from zerver.lib.test_classes import ZulipTestCase + class ZcommandTest(ZulipTestCase): diff --git a/zerver/tests/test_zephyr.py b/zerver/tests/test_zephyr.py index fef809b576bde..dae495d455759 100644 --- a/zerver/tests/test_zephyr.py +++ b/zerver/tests/test_zephyr.py @@ -1,12 +1,12 @@ -import ujson +from typing import Any +from unittest.mock import patch +import ujson from django.http import HttpResponse -from unittest.mock import patch -from typing import Any from zerver.lib.test_classes import ZulipTestCase from zerver.lib.users import get_api_key -from zerver.models import get_user, get_realm +from zerver.models import get_realm, get_user class ZephyrTest(ZulipTestCase): diff --git a/zerver/tornado/application.py b/zerver/tornado/application.py index 019cffbf23403..a303b9a15f251 100644 --- a/zerver/tornado/application.py +++ b/zerver/tornado/application.py @@ -2,11 +2,12 @@ import tornado.web from django.conf import settings -from zerver.tornado import autoreload from zerver.lib.queue import get_queue_client +from zerver.tornado import autoreload from zerver.tornado.handlers import AsyncDjangoHandler + def setup_tornado_rabbitmq() -> None: # nocoverage # When tornado is shut down, disconnect cleanly from rabbitmq if settings.USING_RABBITMQ: diff --git a/zerver/tornado/autoreload.py b/zerver/tornado/autoreload.py index a9d7a98e54796..4aaec8f3876c0 100644 --- a/zerver/tornado/autoreload.py +++ b/zerver/tornado/autoreload.py @@ -50,18 +50,17 @@ # code that didn't work. -import os -import sys import functools import importlib +import os +import subprocess +import sys import traceback import types -import subprocess import weakref -from tornado import ioloop +from tornado import ioloop, process from tornado.log import gen_log -from tornado import process try: import signal diff --git a/zerver/tornado/descriptors.py b/zerver/tornado/descriptors.py index 135ead63bc096..402067a935866 100644 --- a/zerver/tornado/descriptors.py +++ b/zerver/tornado/descriptors.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Dict, Optional if TYPE_CHECKING: from zerver.tornado.event_queue import ClientDescriptor diff --git a/zerver/tornado/event_queue.py b/zerver/tornado/event_queue.py index 4803ff7c40899..2aea57b176ffe 100644 --- a/zerver/tornado/event_queue.py +++ b/zerver/tornado/event_queue.py @@ -1,39 +1,57 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/events-system.html for # high-level documentation on how this system works. -from typing import cast, AbstractSet, Any, Callable, Deque, Dict, List, \ - Mapping, MutableMapping, Optional, Iterable, Sequence, Set, Union -from typing_extensions import TypedDict - -from django.utils.translation import ugettext as _ -from django.conf import settings -from collections import deque -import os -import time -import logging -import ujson -import requests import atexit -import sys +import copy +import logging +import os +import random import signal +import sys +import time import traceback +from collections import deque +from typing import ( + AbstractSet, + Any, + Callable, + Deque, + Dict, + Iterable, + List, + Mapping, + MutableMapping, + Optional, + Sequence, + Set, + Union, + cast, +) + +import requests import tornado.ioloop -import random -from zerver.models import UserProfile, Client, Realm +import ujson +from django.conf import settings +from django.utils.translation import ugettext as _ +from typing_extensions import TypedDict + from zerver.decorator import cachify -from zerver.tornado.handlers import clear_handler_by_id, get_handler_by_id, \ - finish_handler, handler_stats_string -from zerver.lib.utils import statsd -from zerver.middleware import async_request_timer_restart from zerver.lib.message import MessageDict from zerver.lib.narrow import build_narrow_filter from zerver.lib.queue import queue_json_publish, retry_event from zerver.lib.request import JsonableError +from zerver.lib.utils import statsd +from zerver.middleware import async_request_timer_restart +from zerver.models import Client, Realm, UserProfile +from zerver.tornado.autoreload import add_reload_hook from zerver.tornado.descriptors import clear_descriptor_by_handler_id, set_descriptor_by_handler_id from zerver.tornado.exceptions import BadEventQueueIdError -from zerver.tornado.sharding import get_tornado_uri, get_tornado_port, \ - notify_tornado_queue_name -from zerver.tornado.autoreload import add_reload_hook -import copy +from zerver.tornado.handlers import ( + clear_handler_by_id, + finish_handler, + get_handler_by_id, + handler_stats_string, +) +from zerver.tornado.sharding import get_tornado_port, get_tornado_uri, notify_tornado_queue_name requests_client = requests.Session() for host in ['127.0.0.1', 'localhost']: diff --git a/zerver/tornado/exceptions.py b/zerver/tornado/exceptions.py index 652dc579b6e30..9b194e6d78806 100644 --- a/zerver/tornado/exceptions.py +++ b/zerver/tornado/exceptions.py @@ -2,6 +2,7 @@ from zerver.lib.exceptions import ErrorCode, JsonableError + class BadEventQueueIdError(JsonableError): code = ErrorCode.BAD_EVENT_QUEUE_ID data_fields = ['queue_id'] diff --git a/zerver/tornado/handlers.py b/zerver/tornado/handlers.py index e1af1f8a5ca4a..74659ca75750a 100644 --- a/zerver/tornado/handlers.py +++ b/zerver/tornado/handlers.py @@ -7,8 +7,8 @@ from django.core import signals from django.core.handlers import base from django.core.handlers.wsgi import WSGIRequest, get_script_name -from django.urls import set_script_prefix from django.http import HttpRequest, HttpResponse +from django.urls import set_script_prefix from tornado.wsgi import WSGIContainer from zerver.lib.response import json_response diff --git a/zerver/tornado/sharding.py b/zerver/tornado/sharding.py index aeefb66738b8e..7a7cb9588ba9f 100644 --- a/zerver/tornado/sharding.py +++ b/zerver/tornado/sharding.py @@ -1,8 +1,10 @@ +import json +import os + from django.conf import settings from zerver.models import Realm -import json -import os + shard_map = {} if os.path.exists("/etc/zulip/sharding.json"): with open("/etc/zulip/sharding.json") as f: diff --git a/zerver/tornado/views.py b/zerver/tornado/views.py index fb8a24e9f78f2..9de1c1d3f2073 100644 --- a/zerver/tornado/views.py +++ b/zerver/tornado/views.py @@ -5,16 +5,20 @@ from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ -from zerver.decorator import REQ, has_request_variables, internal_notify_view, \ - process_client +from zerver.decorator import REQ, has_request_variables, internal_notify_view, process_client from zerver.lib.response import json_error, json_success -from zerver.lib.validator import check_bool, check_int, check_list, check_string, \ - to_non_negative_int +from zerver.lib.validator import ( + check_bool, + check_int, + check_list, + check_string, + to_non_negative_int, +) from zerver.models import Client, UserProfile, get_client, get_user_profile_by_id -from zerver.tornado.event_queue import fetch_events, \ - get_client_descriptor, process_notification -from zerver.tornado.handlers import AsyncDjangoHandler +from zerver.tornado.event_queue import fetch_events, get_client_descriptor, process_notification from zerver.tornado.exceptions import BadEventQueueIdError +from zerver.tornado.handlers import AsyncDjangoHandler + @internal_notify_view(True) def notify(request: HttpRequest) -> HttpResponse: diff --git a/zerver/views/alert_words.py b/zerver/views/alert_words.py index c4821fd847db6..1f6ba2d23af10 100644 --- a/zerver/views/alert_words.py +++ b/zerver/views/alert_words.py @@ -1,14 +1,14 @@ -from django.http import HttpResponse, HttpRequest - from typing import List -from zerver.models import UserProfile -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.response import json_success -from zerver.lib.validator import check_list, check_string, check_capped_string +from django.http import HttpRequest, HttpResponse from zerver.lib.actions import do_add_alert_words, do_remove_alert_words from zerver.lib.alert_words import user_alert_words +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_success +from zerver.lib.validator import check_capped_string, check_list, check_string +from zerver.models import UserProfile + def list_alert_words(request: HttpRequest, user_profile: UserProfile) -> HttpResponse: return json_success({'alert_words': user_alert_words(user_profile)}) diff --git a/zerver/views/archive.py b/zerver/views/archive.py index 3b610a17fe8d1..fbb7cbad18636 100644 --- a/zerver/views/archive.py +++ b/zerver/views/archive.py @@ -3,17 +3,15 @@ from django.http import HttpRequest, HttpResponse from django.shortcuts import render from django.template import loader -from zerver.lib.streams import get_stream_by_id -from zerver.models import Message, UserProfile from zerver.lib.avatar import get_gravatar_url +from zerver.lib.exceptions import JsonableError from zerver.lib.response import json_success +from zerver.lib.streams import get_stream_by_id from zerver.lib.timestamp import datetime_to_timestamp -from zerver.lib.topic import ( - get_topic_history_for_web_public_stream, - messages_for_topic, -) -from zerver.lib.exceptions import JsonableError +from zerver.lib.topic import get_topic_history_for_web_public_stream, messages_for_topic +from zerver.models import Message, UserProfile + def archive(request: HttpRequest, stream_id: int, diff --git a/zerver/views/attachments.py b/zerver/views/attachments.py index 250c004b875a6..e6a6cace8b3cc 100644 --- a/zerver/views/attachments.py +++ b/zerver/views/attachments.py @@ -1,10 +1,9 @@ from django.http import HttpRequest, HttpResponse -from zerver.models import UserProfile from zerver.lib.actions import notify_attachment_update +from zerver.lib.attachments import access_attachment_by_id, remove_attachment, user_attachments from zerver.lib.response import json_success -from zerver.lib.attachments import user_attachments, remove_attachment, \ - access_attachment_by_id +from zerver.models import UserProfile def list_by_user(request: HttpRequest, user_profile: UserProfile) -> HttpResponse: diff --git a/zerver/views/auth.py b/zerver/views/auth.py index 1fbe546f9fb8d..86fc3148bcbb7 100644 --- a/zerver/views/auth.py +++ b/zerver/views/auth.py @@ -1,39 +1,47 @@ +import logging import os +import urllib +from functools import wraps +from typing import Any, Dict, List, Mapping, Optional, cast + +import jwt from cryptography.hazmat.primitives.ciphers.aead import AESGCM -from django.forms import Form from django.conf import settings from django.contrib.auth import authenticate -from django.contrib.auth.views import LoginView as DjangoLoginView, \ - logout_then_login as django_logout_then_login +from django.contrib.auth.views import LoginView as DjangoLoginView from django.contrib.auth.views import PasswordResetView as DjangoPasswordResetView -from django.urls import reverse -from zerver.decorator import require_post, \ - process_client, do_login, log_view_func -from django.http import HttpRequest, HttpResponse, HttpResponseRedirect, \ - HttpResponseServerError -from django.template.response import SimpleTemplateResponse +from django.contrib.auth.views import logout_then_login as django_logout_then_login +from django.forms import Form +from django.http import HttpRequest, HttpResponse, HttpResponseRedirect, HttpResponseServerError from django.shortcuts import redirect, render +from django.template.response import SimpleTemplateResponse +from django.urls import reverse +from django.utils.http import is_safe_url +from django.utils.translation import ugettext as _ from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_safe from django.views.generic import TemplateView -from django.utils.translation import ugettext as _ -from django.utils.http import is_safe_url -from functools import wraps -import urllib -from typing import Any, Dict, List, Optional, Mapping, cast +from social_django.utils import load_backend, load_strategy +from two_factor.forms import BackupTokenForm +from two_factor.views import LoginView as BaseTwoFactorLoginView from confirmation.models import Confirmation, create_confirmation_link -from zerver.context_processors import zulip_default_context, get_realm_from_request, \ - login_context -from zerver.forms import HomepageForm, OurAuthenticationForm, \ - DEACTIVATED_ACCOUNT_ERROR, ZulipPasswordResetForm, \ - AuthenticationTokenForm +from version import API_FEATURE_LEVEL, ZULIP_VERSION +from zerver.context_processors import get_realm_from_request, login_context, zulip_default_context +from zerver.decorator import do_login, log_view_func, process_client, require_post +from zerver.forms import ( + DEACTIVATED_ACCOUNT_ERROR, + AuthenticationTokenForm, + HomepageForm, + OurAuthenticationForm, + ZulipPasswordResetForm, +) from zerver.lib.mobile_auth_otp import otp_encrypt_api_key from zerver.lib.push_notifications import push_notifications_enabled from zerver.lib.pysa import mark_sanitized from zerver.lib.realm_icon import realm_icon_url -from zerver.lib.request import REQ, has_request_variables, JsonableError -from zerver.lib.response import json_success, json_error +from zerver.lib.request import REQ, JsonableError, has_request_variables +from zerver.lib.response import json_error, json_success from zerver.lib.sessions import set_expirable_session_var from zerver.lib.subdomains import get_subdomain, is_subdomain_root_or_alias from zerver.lib.types import ViewFuncT @@ -42,23 +50,24 @@ from zerver.lib.users import get_api_key from zerver.lib.utils import has_api_key_format from zerver.lib.validator import validate_login_email -from zerver.models import PreregistrationUser, UserProfile, remote_user_to_email, Realm, \ - get_realm +from zerver.models import PreregistrationUser, Realm, UserProfile, get_realm, remote_user_to_email from zerver.signals import email_on_new_login -from zproject.backends import password_auth_enabled, dev_auth_enabled, \ - ldap_auth_enabled, ZulipLDAPConfigurationError, ZulipLDAPAuthBackend, \ - AUTH_BACKEND_NAME_MAP, auth_enabled_helper, saml_auth_enabled, SAMLAuthBackend, \ - redirect_to_config_error, ZulipRemoteUserBackend, validate_otp_params, ExternalAuthResult, \ - ExternalAuthDataDict -from version import ZULIP_VERSION, API_FEATURE_LEVEL - -import jwt -import logging - -from social_django.utils import load_backend, load_strategy - -from two_factor.forms import BackupTokenForm -from two_factor.views import LoginView as BaseTwoFactorLoginView +from zproject.backends import ( + AUTH_BACKEND_NAME_MAP, + ExternalAuthDataDict, + ExternalAuthResult, + SAMLAuthBackend, + ZulipLDAPAuthBackend, + ZulipLDAPConfigurationError, + ZulipRemoteUserBackend, + auth_enabled_helper, + dev_auth_enabled, + ldap_auth_enabled, + password_auth_enabled, + redirect_to_config_error, + saml_auth_enabled, + validate_otp_params, +) ExtraContext = Optional[Dict[str, Any]] diff --git a/zerver/views/camo.py b/zerver/views/camo.py index fecbdd868e8c9..414eb6f6020c6 100644 --- a/zerver/views/camo.py +++ b/zerver/views/camo.py @@ -1,13 +1,13 @@ +import binascii + from django.conf import settings +from django.http import HttpRequest, HttpResponse, HttpResponseForbidden, HttpResponseNotFound from django.shortcuts import redirect from django.utils.translation import ugettext as _ -from django.http import ( - HttpRequest, HttpResponse, HttpResponseForbidden, HttpResponseNotFound, -) + from zerver.lib.camo import is_camo_url_valid from zerver.lib.thumbnail import generate_thumbnail_url -import binascii def handle_camo_url(request: HttpRequest, digest: str, received_url: str) -> HttpResponse: diff --git a/zerver/views/compatibility.py b/zerver/views/compatibility.py index 6c0e08b158e6b..55b9d4890e34f 100644 --- a/zerver/views/compatibility.py +++ b/zerver/views/compatibility.py @@ -1,13 +1,14 @@ -from django.http import HttpResponse, HttpRequest import re from typing import List, Optional, Tuple +from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ +from version import DESKTOP_MINIMUM_VERSION, DESKTOP_WARNING_VERSION from zerver.lib.response import json_error, json_success from zerver.lib.user_agent import parse_user_agent from zerver.signals import get_device_browser -from version import DESKTOP_MINIMUM_VERSION, DESKTOP_WARNING_VERSION + def pop_numerals(ver: str) -> Tuple[List[int], str]: match = re.search(r'^( \d+ (?: \. \d+ )* ) (.*)', ver, re.X) diff --git a/zerver/views/custom_profile_fields.py b/zerver/views/custom_profile_fields.py index 850a11165fdea..96941837efa29 100644 --- a/zerver/views/custom_profile_fields.py +++ b/zerver/views/custom_profile_fields.py @@ -1,29 +1,35 @@ -from typing import Union, List, Dict -import ujson +from typing import Dict, List, Union +import ujson from django.db import IntegrityError from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ -from zerver.decorator import require_realm_admin, human_users_only -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.actions import (try_add_realm_custom_profile_field, - do_remove_realm_custom_profile_field, - try_update_realm_custom_profile_field, - do_update_user_custom_profile_data_if_changed, - try_reorder_realm_custom_profile_fields, - try_add_realm_default_custom_profile_field, - check_remove_custom_profile_field_value) -from zerver.lib.response import json_success, json_error -from zerver.lib.types import ProfileFieldData -from zerver.lib.validator import (check_dict, check_list, check_int, - validate_choice_field_data, check_capped_string) - -from zerver.models import (UserProfile, - CustomProfileField, custom_profile_fields_for_realm) +from zerver.decorator import human_users_only, require_realm_admin +from zerver.lib.actions import ( + check_remove_custom_profile_field_value, + do_remove_realm_custom_profile_field, + do_update_user_custom_profile_data_if_changed, + try_add_realm_custom_profile_field, + try_add_realm_default_custom_profile_field, + try_reorder_realm_custom_profile_fields, + try_update_realm_custom_profile_field, +) from zerver.lib.exceptions import JsonableError -from zerver.lib.users import validate_user_custom_profile_data from zerver.lib.external_accounts import validate_external_account_field_data +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_error, json_success +from zerver.lib.types import ProfileFieldData +from zerver.lib.users import validate_user_custom_profile_data +from zerver.lib.validator import ( + check_capped_string, + check_dict, + check_int, + check_list, + validate_choice_field_data, +) +from zerver.models import CustomProfileField, UserProfile, custom_profile_fields_for_realm + def list_realm_custom_profile_fields(request: HttpRequest, user_profile: UserProfile) -> HttpResponse: fields = custom_profile_fields_for_realm(user_profile.realm_id) diff --git a/zerver/views/development/email_log.py b/zerver/views/development/email_log.py index 3cab8c4c29c6c..bbfcbff3447c5 100755 --- a/zerver/views/development/email_log.py +++ b/zerver/views/development/email_log.py @@ -1,25 +1,19 @@ +import os +import subprocess +import urllib + +import ujson from django.conf import settings from django.http import HttpRequest, HttpResponse -from django.shortcuts import render, redirect +from django.shortcuts import redirect, render from django.views.decorators.http import require_safe -from zerver.models import ( - get_realm, get_user_by_delivery_email, get_realm_stream, Realm, -) +from confirmation.models import Confirmation, confirmation_url +from zerver.lib.actions import do_change_user_delivery_email, do_send_realm_reactivation_email from zerver.lib.email_notifications import enqueue_welcome_emails from zerver.lib.response import json_success -from zerver.lib.actions import do_change_user_delivery_email, \ - do_send_realm_reactivation_email -from zproject.email_backends import ( - get_forward_address, - set_forward_address, -) -import urllib -from confirmation.models import Confirmation, confirmation_url - -import os -import subprocess -import ujson +from zerver.models import Realm, get_realm, get_realm_stream, get_user_by_delivery_email +from zproject.email_backends import get_forward_address, set_forward_address ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../') diff --git a/zerver/views/development/integrations.py b/zerver/views/development/integrations.py index 9da34f13f9c76..0b5701338a0f9 100644 --- a/zerver/views/development/integrations.py +++ b/zerver/views/development/integrations.py @@ -1,19 +1,17 @@ import os -import ujson from typing import Any, Dict, List, Optional +import ujson from django.http import HttpRequest, HttpResponse from django.shortcuts import render from django.test import Client from zerver.lib.integrations import WEBHOOK_INTEGRATIONS -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.response import json_success, json_error -from zerver.models import UserProfile, get_realm +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_error, json_success from zerver.lib.validator import check_bool -from zerver.lib.webhooks.common import get_fixture_http_headers, \ - standardize_headers - +from zerver.lib.webhooks.common import get_fixture_http_headers, standardize_headers +from zerver.models import UserProfile, get_realm ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../') diff --git a/zerver/views/development/registration.py b/zerver/views/development/registration.py index ee8a8047ba2b0..467ea77605635 100644 --- a/zerver/views/development/registration.py +++ b/zerver/views/development/registration.py @@ -1,14 +1,13 @@ +from typing import Any + from django.conf import settings -from django.http import HttpResponse, HttpRequest +from django.http import HttpRequest, HttpResponse from django.views.decorators.csrf import csrf_exempt from confirmation.models import Confirmation, create_confirmation_link - -from typing import Any - -from zerver.models import UserProfile from zerver.lib.response import json_success from zerver.lib.subdomains import get_subdomain +from zerver.models import UserProfile from zerver.views.auth import create_preregistration_user from zerver.views.registration import accounts_register diff --git a/zerver/views/digest.py b/zerver/views/digest.py index 524755f79c7de..d9eb84762f9df 100644 --- a/zerver/views/digest.py +++ b/zerver/views/digest.py @@ -1,12 +1,14 @@ import time +from datetime import timedelta + +from django.conf import settings from django.http import HttpRequest, HttpResponse from django.shortcuts import render from django.utils.timezone import now as timezone_now -from django.conf import settings -from zerver.lib.digest import handle_digest_email, DIGEST_CUTOFF from zerver.decorator import zulip_login_required -from datetime import timedelta +from zerver.lib.digest import DIGEST_CUTOFF, handle_digest_email + @zulip_login_required def digest_page(request: HttpRequest) -> HttpResponse: diff --git a/zerver/views/documentation.py b/zerver/views/documentation.py index f26c49b4bdffe..e93f04ce81719 100644 --- a/zerver/views/documentation.py +++ b/zerver/views/documentation.py @@ -1,22 +1,22 @@ -from typing import Any, Dict, Tuple +import os +import random +import re from collections import OrderedDict -from django.views.generic import TemplateView +from typing import Any, Dict, Tuple + from django.conf import settings from django.http import HttpRequest, HttpResponse, HttpResponseNotFound from django.template import loader +from django.views.generic import TemplateView -import os -import random -import re - +from zerver.context_processors import zulip_default_context from zerver.decorator import add_google_analytics_context -from zerver.lib.integrations import CATEGORIES, INTEGRATIONS, HubotIntegration, \ - WebhookIntegration -from zerver.lib.request import has_request_variables, REQ +from zerver.lib.integrations import CATEGORIES, INTEGRATIONS, HubotIntegration, WebhookIntegration +from zerver.lib.request import REQ, has_request_variables from zerver.lib.subdomains import get_subdomain from zerver.models import Realm from zerver.templatetags.app_filters import render_markdown_path -from zerver.context_processors import zulip_default_context + def add_api_uri_context(context: Dict[str, Any], request: HttpRequest) -> None: context.update(zulip_default_context(request)) diff --git a/zerver/views/email_mirror.py b/zerver/views/email_mirror.py index 8e8260111c788..1f04e7f2c6f5d 100644 --- a/zerver/views/email_mirror.py +++ b/zerver/views/email_mirror.py @@ -1,11 +1,11 @@ -import ujson +from typing import Dict +import ujson from django.http import HttpRequest, HttpResponse -from typing import Dict from zerver.decorator import internal_notify_view from zerver.lib.email_mirror import mirror_email_message -from zerver.lib.request import has_request_variables, REQ +from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_error, json_success from zerver.lib.validator import check_dict, check_string diff --git a/zerver/views/events_register.py b/zerver/views/events_register.py index eab0f35a05053..32b3277877205 100644 --- a/zerver/views/events_register.py +++ b/zerver/views/events_register.py @@ -1,12 +1,14 @@ -from django.http import HttpRequest, HttpResponse from typing import Dict, Iterable, Optional, Sequence +from django.http import HttpRequest, HttpResponse + from zerver.lib.events import do_events_register from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.validator import check_dict, check_string, check_list, check_bool +from zerver.lib.validator import check_bool, check_dict, check_list, check_string from zerver.models import Stream, UserProfile + def _default_all_public_streams(user_profile: UserProfile, all_public_streams: Optional[bool]) -> bool: if all_public_streams is not None: diff --git a/zerver/views/home.py b/zerver/views/home.py index 5f15d5ddf8b6b..a072ea2467403 100644 --- a/zerver/views/home.py +++ b/zerver/views/home.py @@ -1,35 +1,36 @@ -from typing import Any, List, Dict, Optional, Tuple +import calendar +import logging +import time +from typing import Any, Dict, List, Optional, Tuple from django.conf import settings -from django.urls import reverse -from django.http import HttpResponseRedirect, HttpResponse, HttpRequest +from django.http import HttpRequest, HttpResponse, HttpResponseRedirect from django.shortcuts import redirect, render +from django.urls import reverse from django.utils import translation from django.utils.cache import patch_cache_control +from two_factor.utils import default_device from zerver.decorator import zulip_login_required from zerver.forms import ToSForm -from zerver.models import Message, Stream, UserProfile, \ - Realm, PreregistrationUser +from zerver.lib.actions import do_change_tos_version, realm_user_count from zerver.lib.events import do_events_register -from zerver.lib.actions import do_change_tos_version, \ - realm_user_count -from zerver.lib.i18n import get_language_list, get_language_name, \ - get_language_list_for_templates, get_language_translation_data +from zerver.lib.i18n import ( + get_language_list, + get_language_list_for_templates, + get_language_name, + get_language_translation_data, +) from zerver.lib.push_notifications import num_push_devices_for_user from zerver.lib.streams import access_stream_by_name from zerver.lib.subdomains import get_subdomain from zerver.lib.users import compute_show_invites_and_add_streams -from zerver.lib.utils import statsd, generate_random_token -from zerver.views.compatibility import is_outdated_desktop_app, \ - is_unsupported_browser +from zerver.lib.utils import generate_random_token, statsd +from zerver.models import Message, PreregistrationUser, Realm, Stream, UserProfile +from zerver.views.compatibility import is_outdated_desktop_app, is_unsupported_browser from zerver.views.messages import get_latest_update_message_flag_activity from zerver.views.portico import hello_view -from two_factor.utils import default_device -import calendar -import logging -import time def need_accept_tos(user_profile: Optional[UserProfile]) -> bool: if user_profile is None: # nocoverage diff --git a/zerver/views/hotspots.py b/zerver/views/hotspots.py index 7efa693850989..929ee2692e1e6 100644 --- a/zerver/views/hotspots.py +++ b/zerver/views/hotspots.py @@ -4,11 +4,12 @@ from zerver.decorator import human_users_only from zerver.lib.actions import do_mark_hotspot_as_read from zerver.lib.hotspots import ALL_HOTSPOTS -from zerver.lib.request import has_request_variables, REQ +from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_error, json_success from zerver.lib.validator import check_string from zerver.models import UserProfile + @human_users_only @has_request_variables def mark_hotspot_as_read(request: HttpRequest, user: UserProfile, diff --git a/zerver/views/invite.py b/zerver/views/invite.py index 25e4c5db8d2af..de93ac5d98927 100644 --- a/zerver/views/invite.py +++ b/zerver/views/invite.py @@ -1,19 +1,25 @@ +import re +from typing import List, Optional, Set + from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ -from typing import List, Optional, Set -from zerver.decorator import require_realm_admin, require_member_or_admin -from zerver.lib.actions import do_invite_users, do_revoke_user_invite, \ - do_revoke_multi_use_invite, do_resend_user_invite_email, \ - do_get_user_invites, do_create_multiuse_invite_link +from zerver.decorator import require_member_or_admin, require_realm_admin +from zerver.lib.actions import ( + do_create_multiuse_invite_link, + do_get_user_invites, + do_invite_users, + do_resend_user_invite_email, + do_revoke_multi_use_invite, + do_revoke_user_invite, +) from zerver.lib.exceptions import OrganizationAdministratorRequired -from zerver.lib.request import REQ, has_request_variables, JsonableError -from zerver.lib.response import json_success, json_error +from zerver.lib.request import REQ, JsonableError, has_request_variables +from zerver.lib.response import json_error, json_success from zerver.lib.streams import access_stream_by_id -from zerver.lib.validator import check_list, check_int -from zerver.models import PreregistrationUser, Stream, UserProfile, MultiuseInvite +from zerver.lib.validator import check_int, check_list +from zerver.models import MultiuseInvite, PreregistrationUser, Stream, UserProfile -import re @require_member_or_admin @has_request_variables diff --git a/zerver/views/messages.py b/zerver/views/messages.py index a031d3e2ff7c7..6ad5a5b34b6b2 100644 --- a/zerver/views/messages.py +++ b/zerver/views/messages.py @@ -1,70 +1,119 @@ -from django.utils.translation import ugettext as _ -from django.utils.timezone import now as timezone_now +import datetime +import re +from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union, cast + +import ujson +from dateutil.parser import parse as dateparser from django.conf import settings from django.core import validators from django.core.exceptions import ValidationError -from django.db import connection, IntegrityError +from django.db import IntegrityError, connection from django.http import HttpRequest, HttpResponse -from typing import Dict, List, Set, Any, Iterable, \ - Optional, Tuple, Union, Sequence, cast -from zerver.lib.exceptions import JsonableError, ErrorCode -from zerver.lib.html_diff import highlight_html_differences -from zerver.decorator import has_request_variables, REQ from django.utils.html import escape as escape_html +from django.utils.timezone import now as timezone_now +from django.utils.translation import ugettext as _ +from sqlalchemy import func +from sqlalchemy.dialects import postgresql +from sqlalchemy.sql import ( + ColumnElement, + Selectable, + alias, + and_, + column, + join, + literal, + literal_column, + not_, + or_, + select, + table, + union_all, +) + +from zerver.decorator import REQ, has_request_variables from zerver.lib import bugdown -from zerver.lib.zcommand import process_zcommands -from zerver.lib.actions import recipient_for_user_profiles, do_update_message_flags, \ - compute_irc_user_fullname, compute_jabber_user_fullname, \ - create_mirror_user_if_needed, check_send_message, do_update_message, \ - extract_private_recipients, render_incoming_message, do_delete_messages, \ - do_mark_all_as_read, do_mark_stream_messages_as_read, extract_stream_indicator, \ - get_user_info_for_message_updates, check_schedule_message +from zerver.lib.actions import ( + check_schedule_message, + check_send_message, + compute_irc_user_fullname, + compute_jabber_user_fullname, + create_mirror_user_if_needed, + do_delete_messages, + do_mark_all_as_read, + do_mark_stream_messages_as_read, + do_update_message, + do_update_message_flags, + extract_private_recipients, + extract_stream_indicator, + get_user_info_for_message_updates, + recipient_for_user_profiles, + render_incoming_message, +) from zerver.lib.addressee import get_user_profiles, get_user_profiles_by_ids -from zerver.lib.queue import queue_json_publish +from zerver.lib.exceptions import ErrorCode, JsonableError +from zerver.lib.html_diff import highlight_html_differences from zerver.lib.message import ( access_message, + get_first_visible_message_id, messages_for_ids, render_markdown, - get_first_visible_message_id, truncate_body, ) -from zerver.lib.response import json_success, json_error +from zerver.lib.queue import queue_json_publish +from zerver.lib.response import json_error, json_success from zerver.lib.sqlalchemy_utils import get_sqlalchemy_connection -from zerver.lib.streams import access_stream_by_id, get_public_streams_queryset, \ - can_access_stream_history_by_name, can_access_stream_history_by_id, \ - get_stream_by_narrow_operand_access_unchecked, get_stream_by_id -from zerver.lib.timestamp import datetime_to_timestamp, convert_to_UTC +from zerver.lib.streams import ( + access_stream_by_id, + can_access_stream_history_by_id, + can_access_stream_history_by_name, + get_public_streams_queryset, + get_stream_by_id, + get_stream_by_narrow_operand_access_unchecked, +) +from zerver.lib.timestamp import convert_to_UTC, datetime_to_timestamp from zerver.lib.timezone import get_timezone from zerver.lib.topic import ( - topic_column_sa, - topic_match_sa, - user_message_exists_for_topic, DB_TOPIC_NAME, LEGACY_PREV_TOPIC, MATCH_TOPIC, REQ_topic, + topic_column_sa, + topic_match_sa, + user_message_exists_for_topic, ) from zerver.lib.topic_mutes import exclude_topic_mutes from zerver.lib.utils import statsd -from zerver.lib.validator import \ - check_list, check_int, check_dict, check_string, check_bool, \ - check_string_or_int_list, check_string_or_int, check_string_in, \ - check_required_string, to_non_negative_int +from zerver.lib.validator import ( + check_bool, + check_dict, + check_int, + check_list, + check_required_string, + check_string, + check_string_in, + check_string_or_int, + check_string_or_int_list, + to_non_negative_int, +) +from zerver.lib.zcommand import process_zcommands from zerver.lib.zephyr import compute_mit_user_fullname -from zerver.models import Message, UserProfile, Stream, Subscription, Client,\ - Realm, RealmDomain, Recipient, UserMessage, UserActivity, \ - email_to_domain, get_realm, get_active_streams, get_user_including_cross_realm, \ - get_user_by_id_in_realm_including_cross_realm - -from sqlalchemy import func -from sqlalchemy.dialects import postgresql -from sqlalchemy.sql import select, join, column, literal_column, literal, and_, \ - or_, not_, union_all, alias, Selectable, ColumnElement, table - -from dateutil.parser import parse as dateparser -import re -import ujson -import datetime +from zerver.models import ( + Client, + Message, + Realm, + RealmDomain, + Recipient, + Stream, + Subscription, + UserActivity, + UserMessage, + UserProfile, + email_to_domain, + get_active_streams, + get_realm, + get_user_by_id_in_realm_including_cross_realm, + get_user_including_cross_realm, +) LARGER_THAN_MAX_MESSAGE_ID = 10000000000000000 MAX_MESSAGES_PER_FETCH = 5000 diff --git a/zerver/views/muting.py b/zerver/views/muting.py index 77dbfdcd756cf..c4f46c5aee56e 100644 --- a/zerver/views/muting.py +++ b/zerver/views/muting.py @@ -1,13 +1,13 @@ -from django.http import HttpResponse, HttpRequest -from typing import Optional import datetime +from typing import Optional +from django.http import HttpRequest, HttpResponse from django.utils.timezone import now as timezone_now from django.utils.translation import ugettext as _ + from zerver.lib.actions import do_mute_topic, do_unmute_topic -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.response import json_success, json_error -from zerver.lib.topic_mutes import topic_is_muted +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_error, json_success from zerver.lib.streams import ( access_stream_by_id, access_stream_by_name, @@ -15,9 +15,11 @@ access_stream_for_unmute_topic_by_name, check_for_exactly_one_stream_arg, ) +from zerver.lib.topic_mutes import topic_is_muted from zerver.lib.validator import check_int from zerver.models import UserProfile + def mute_topic(user_profile: UserProfile, stream_id: Optional[int], stream_name: Optional[str], diff --git a/zerver/views/pointer.py b/zerver/views/pointer.py index 924382098d41b..08e9a0756e445 100644 --- a/zerver/views/pointer.py +++ b/zerver/views/pointer.py @@ -2,11 +2,12 @@ from django.utils.translation import ugettext as _ from zerver.lib.actions import do_update_pointer -from zerver.lib.request import has_request_variables, JsonableError, REQ +from zerver.lib.request import REQ, JsonableError, has_request_variables from zerver.lib.response import json_success from zerver.lib.validator import to_non_negative_int from zerver.models import UserProfile, get_usermessage_by_message_id + def get_pointer_backend(request: HttpRequest, user_profile: UserProfile) -> HttpResponse: return json_success({'pointer': user_profile.pointer}) diff --git a/zerver/views/portico.py b/zerver/views/portico.py index b6fdc13706c01..770dc7eaebab8 100644 --- a/zerver/views/portico.py +++ b/zerver/views/portico.py @@ -1,12 +1,13 @@ +import ujson from django.conf import settings from django.http import HttpRequest, HttpResponse, HttpResponseRedirect from django.template.response import TemplateResponse -import ujson +from version import LATEST_DESKTOP_VERSION from zerver.context_processors import get_realm_from_request, latest_info_context from zerver.decorator import add_google_analytics, redirect_to_login from zerver.models import Realm -from version import LATEST_DESKTOP_VERSION + @add_google_analytics def apps_view(request: HttpRequest, _: str) -> HttpResponse: diff --git a/zerver/views/presence.py b/zerver/views/presence.py index 76525c7f5a2dc..7d7e23d6bed86 100644 --- a/zerver/views/presence.py +++ b/zerver/views/presence.py @@ -1,27 +1,20 @@ import datetime - -from django.conf import settings from typing import Any, Dict, Optional +from django.conf import settings from django.http import HttpRequest, HttpResponse from django.utils.timezone import now as timezone_now from django.utils.translation import ugettext as _ from zerver.decorator import human_users_only -from zerver.lib.actions import ( - do_update_user_status, - update_user_presence, -) -from zerver.lib.presence import ( - get_presence_response, - get_presence_for_user, -) -from zerver.lib.request import has_request_variables, REQ, JsonableError -from zerver.lib.response import json_success, json_error +from zerver.lib.actions import do_update_user_status, update_user_presence +from zerver.lib.presence import get_presence_for_user, get_presence_response +from zerver.lib.request import REQ, JsonableError, has_request_variables +from zerver.lib.response import json_error, json_success from zerver.lib.timestamp import datetime_to_timestamp from zerver.lib.validator import check_bool, check_capped_string -from zerver.models import UserActivity, UserPresence, UserProfile, \ - get_active_user +from zerver.models import UserActivity, UserPresence, UserProfile, get_active_user + def get_presence_backend(request: HttpRequest, user_profile: UserProfile, email: str) -> HttpResponse: diff --git a/zerver/views/push_notifications.py b/zerver/views/push_notifications.py index 2d4d88e3ad522..67083211c71ea 100644 --- a/zerver/views/push_notifications.py +++ b/zerver/views/push_notifications.py @@ -3,12 +3,16 @@ from django.utils.translation import ugettext as _ from zerver.decorator import human_users_only -from zerver.lib.push_notifications import add_push_device_token, \ - b64_to_hex, remove_push_device_token -from zerver.lib.request import has_request_variables, REQ, JsonableError +from zerver.lib.push_notifications import ( + add_push_device_token, + b64_to_hex, + remove_push_device_token, +) +from zerver.lib.request import REQ, JsonableError, has_request_variables from zerver.lib.response import json_success from zerver.models import PushDeviceToken, UserProfile + def validate_token(token_str: str, kind: int) -> None: if token_str == '' or len(token_str) > 4096: raise JsonableError(_('Empty or invalid length token')) diff --git a/zerver/views/reactions.py b/zerver/views/reactions.py index 31180adddb71c..951ad88a4fc85 100644 --- a/zerver/views/reactions.py +++ b/zerver/views/reactions.py @@ -1,8 +1,9 @@ +from typing import Optional + from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ -from zerver.decorator import \ - has_request_variables, REQ +from zerver.decorator import REQ, has_request_variables from zerver.lib.actions import do_add_reaction, do_remove_reaction from zerver.lib.emoji import check_emoji_request, emoji_name_to_emoji_code from zerver.lib.message import access_message @@ -10,7 +11,6 @@ from zerver.lib.response import json_success from zerver.models import Message, Reaction, UserMessage, UserProfile -from typing import Optional def create_historical_message(user_profile: UserProfile, message: Message) -> None: # Users can see and react to messages sent to streams they diff --git a/zerver/views/realm.py b/zerver/views/realm.py index 40ec58ff1f4cb..3dc1ecb8efa86 100644 --- a/zerver/views/realm.py +++ b/zerver/views/realm.py @@ -1,31 +1,40 @@ from typing import Any, Dict, Optional + +from django.core.exceptions import ValidationError from django.http import HttpRequest, HttpResponse from django.shortcuts import render from django.utils.translation import ugettext as _ -from django.core.exceptions import ValidationError from django.views.decorators.http import require_safe +from confirmation.models import Confirmation, ConfirmationKeyException, get_object_from_key from zerver.decorator import require_realm_admin, require_realm_owner +from zerver.forms import check_subdomain_available as check_subdomain from zerver.lib.actions import ( - do_set_realm_message_editing, - do_set_realm_message_deleting, + do_deactivate_realm, + do_reactivate_realm, do_set_realm_authentication_methods, + do_set_realm_message_deleting, + do_set_realm_message_editing, do_set_realm_notifications_stream, - do_set_realm_signup_notifications_stream, do_set_realm_property, - do_deactivate_realm, - do_reactivate_realm, + do_set_realm_signup_notifications_stream, ) +from zerver.lib.domains import validate_domain from zerver.lib.i18n import get_available_language_codes -from zerver.lib.request import has_request_variables, REQ, JsonableError -from zerver.lib.response import json_success, json_error -from zerver.lib.validator import check_string, check_dict, check_bool, check_int, \ - check_int_in, to_positive_or_allowed_int, to_non_negative_int +from zerver.lib.request import REQ, JsonableError, has_request_variables +from zerver.lib.response import json_error, json_success from zerver.lib.streams import access_stream_by_id -from zerver.lib.domains import validate_domain +from zerver.lib.validator import ( + check_bool, + check_dict, + check_int, + check_int_in, + check_string, + to_non_negative_int, + to_positive_or_allowed_int, +) from zerver.models import Realm, UserProfile -from zerver.forms import check_subdomain_available as check_subdomain -from confirmation.models import get_object_from_key, Confirmation, ConfirmationKeyException + @require_realm_admin @has_request_variables diff --git a/zerver/views/realm_domains.py b/zerver/views/realm_domains.py index 85d014226ac36..60d682ccb7e46 100644 --- a/zerver/views/realm_domains.py +++ b/zerver/views/realm_domains.py @@ -3,10 +3,9 @@ from django.utils.translation import ugettext as _ from zerver.decorator import require_realm_admin -from zerver.lib.actions import do_add_realm_domain, do_change_realm_domain, \ - do_remove_realm_domain +from zerver.lib.actions import do_add_realm_domain, do_change_realm_domain, do_remove_realm_domain from zerver.lib.domains import validate_domain -from zerver.lib.request import has_request_variables, REQ +from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_error, json_success from zerver.lib.validator import check_bool, check_string from zerver.models import RealmDomain, UserProfile, get_realm_domains diff --git a/zerver/views/realm_emoji.py b/zerver/views/realm_emoji.py index 526d70f0909e5..0758caceaa4bf 100644 --- a/zerver/views/realm_emoji.py +++ b/zerver/views/realm_emoji.py @@ -2,12 +2,12 @@ from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ -from zerver.models import RealmEmoji, UserProfile -from zerver.lib.emoji import check_emoji_admin, check_valid_emoji_name -from zerver.lib.request import JsonableError, REQ, has_request_variables -from zerver.lib.response import json_success, json_error -from zerver.lib.actions import check_add_realm_emoji, do_remove_realm_emoji from zerver.decorator import require_member_or_admin +from zerver.lib.actions import check_add_realm_emoji, do_remove_realm_emoji +from zerver.lib.emoji import check_emoji_admin, check_valid_emoji_name +from zerver.lib.request import REQ, JsonableError, has_request_variables +from zerver.lib.response import json_error, json_success +from zerver.models import RealmEmoji, UserProfile def list_emoji(request: HttpRequest, user_profile: UserProfile) -> HttpResponse: diff --git a/zerver/views/realm_export.py b/zerver/views/realm_export.py index 4b7b777fc864d..29d7bdcc58749 100644 --- a/zerver/views/realm_export.py +++ b/zerver/views/realm_export.py @@ -1,20 +1,19 @@ from datetime import timedelta -from analytics.models import RealmCount - +import ujson from django.conf import settings +from django.http import HttpRequest, HttpResponse from django.utils.timezone import now as timezone_now from django.utils.translation import ugettext as _ -from django.http import HttpResponse, HttpRequest +from analytics.models import RealmCount from zerver.decorator import require_realm_admin -from zerver.models import RealmAuditLog, UserProfile +from zerver.lib.actions import do_delete_realm_export, notify_realm_export +from zerver.lib.export import get_realm_exports_serialized from zerver.lib.queue import queue_json_publish from zerver.lib.response import json_error, json_success -from zerver.lib.export import get_realm_exports_serialized -from zerver.lib.actions import do_delete_realm_export, notify_realm_export +from zerver.models import RealmAuditLog, UserProfile -import ujson @require_realm_admin def export_realm(request: HttpRequest, user: UserProfile) -> HttpResponse: diff --git a/zerver/views/realm_filters.py b/zerver/views/realm_filters.py index 625ef7a173eac..366cb66bdb4fd 100644 --- a/zerver/views/realm_filters.py +++ b/zerver/views/realm_filters.py @@ -4,9 +4,9 @@ from zerver.decorator import require_realm_admin from zerver.lib.actions import do_add_realm_filter, do_remove_realm_filter -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.response import json_success, json_error -from zerver.models import realm_filters_for_realm, UserProfile, RealmFilter +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_error, json_success +from zerver.models import RealmFilter, UserProfile, realm_filters_for_realm # Custom realm filters diff --git a/zerver/views/realm_icon.py b/zerver/views/realm_icon.py index 66054248dd2ac..70dff5711bfc2 100644 --- a/zerver/views/realm_icon.py +++ b/zerver/views/realm_icon.py @@ -1,7 +1,7 @@ from django.conf import settings +from django.http import HttpRequest, HttpResponse from django.shortcuts import redirect from django.utils.translation import ugettext as _ -from django.http import HttpResponse, HttpRequest from zerver.decorator import require_realm_admin from zerver.lib.actions import do_change_icon_source diff --git a/zerver/views/realm_logo.py b/zerver/views/realm_logo.py index b7ab5ef03e4a6..ea09ff8cd0c9e 100644 --- a/zerver/views/realm_logo.py +++ b/zerver/views/realm_logo.py @@ -1,16 +1,16 @@ from django.conf import settings +from django.http import HttpRequest, HttpResponse from django.shortcuts import redirect from django.utils.translation import ugettext as _ -from django.http import HttpResponse, HttpRequest -from zerver.lib.validator import check_bool -from zerver.lib.request import REQ, has_request_variables from zerver.decorator import require_realm_admin from zerver.lib.actions import do_change_logo_source from zerver.lib.realm_logo import get_realm_logo_url +from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_error, json_success from zerver.lib.upload import upload_logo_image from zerver.lib.url_encoding import add_query_arg_to_redirect_url +from zerver.lib.validator import check_bool from zerver.models import UserProfile diff --git a/zerver/views/registration.py b/zerver/views/registration.py index eb7dae16fbe08..290acf4cb469b 100644 --- a/zerver/views/registration.py +++ b/zerver/views/registration.py @@ -1,55 +1,92 @@ -from typing import List, Dict, Optional +import logging +import smtplib +import urllib +from typing import Dict, List, Optional -from django.utils.translation import ugettext as _ from django.conf import settings from django.contrib.auth import authenticate, get_backends -from django.urls import reverse -from django.http import HttpResponseRedirect, HttpResponse, HttpRequest -from django.shortcuts import redirect, render -from django.core.exceptions import ValidationError from django.core import validators -from zerver.context_processors import get_realm_from_request, login_context -from zerver.models import UserProfile, Realm, Stream, MultiuseInvite, \ - name_changes_disabled, email_to_username, \ - get_realm, get_user_by_delivery_email, get_default_stream_groups, DisposableEmailError, \ - DomainNotAllowedForRealmError, get_source_profile, EmailContainsPlusError -from zerver.lib.email_validation import email_allowed_for_realm, \ - validate_email_not_already_in_realm -from zerver.lib.send_email import send_email, FromAddress -from zerver.lib.actions import do_change_password, do_change_full_name, \ - do_activate_user, do_create_user, do_create_realm, \ - do_set_user_display_setting, lookup_default_stream_groups, bulk_add_subscriptions -from zerver.forms import RegistrationForm, HomepageForm, RealmCreationForm, \ - FindMyTeamForm, RealmRedirectForm +from django.core.exceptions import ValidationError +from django.http import HttpRequest, HttpResponse, HttpResponseRedirect +from django.shortcuts import redirect, render +from django.urls import reverse +from django.utils.translation import ugettext as _ from django_auth_ldap.backend import LDAPBackend, _LDAPUser -from zerver.decorator import require_post, \ - do_login + +from confirmation import settings as confirmation_settings +from confirmation.models import ( + Confirmation, + ConfirmationKeyException, + RealmCreationKey, + create_confirmation_link, + get_object_from_key, + render_confirmation_key_error, + validate_key, +) +from zerver.context_processors import get_realm_from_request, login_context +from zerver.decorator import do_login, require_post +from zerver.forms import ( + FindMyTeamForm, + HomepageForm, + RealmCreationForm, + RealmRedirectForm, + RegistrationForm, +) +from zerver.lib.actions import ( + bulk_add_subscriptions, + do_activate_user, + do_change_full_name, + do_change_password, + do_create_realm, + do_create_user, + do_set_user_display_setting, + lookup_default_stream_groups, +) from zerver.lib.create_user import get_role_for_new_user +from zerver.lib.email_validation import email_allowed_for_realm, validate_email_not_already_in_realm from zerver.lib.onboarding import send_initial_realm_messages, setup_realm_internal_bots -from zerver.lib.sessions import get_expirable_session_var from zerver.lib.pysa import mark_sanitized +from zerver.lib.send_email import FromAddress, send_email +from zerver.lib.sessions import get_expirable_session_var from zerver.lib.subdomains import get_subdomain, is_root_domain_available from zerver.lib.timezone import get_all_timezones from zerver.lib.url_encoding import add_query_to_redirect_url from zerver.lib.users import get_accounts_for_email from zerver.lib.zephyr import compute_mit_user_fullname -from zerver.views.auth import create_preregistration_user, redirect_and_log_into_subdomain, \ - redirect_to_deactivation_notice, get_safe_redirect_to, finish_desktop_flow, finish_mobile_flow - -from zproject.backends import ldap_auth_enabled, password_auth_enabled, \ - ZulipLDAPExceptionNoMatchingLDAPUser, email_auth_enabled, ZulipLDAPAuthBackend, \ - email_belongs_to_ldap, any_social_backend_enabled, ExternalAuthResult - -from confirmation.models import Confirmation, RealmCreationKey, ConfirmationKeyException, \ - validate_key, create_confirmation_link, get_object_from_key, \ - render_confirmation_key_error +from zerver.models import ( + DisposableEmailError, + DomainNotAllowedForRealmError, + EmailContainsPlusError, + MultiuseInvite, + Realm, + Stream, + UserProfile, + email_to_username, + get_default_stream_groups, + get_realm, + get_source_profile, + get_user_by_delivery_email, + name_changes_disabled, +) +from zerver.views.auth import ( + create_preregistration_user, + finish_desktop_flow, + finish_mobile_flow, + get_safe_redirect_to, + redirect_and_log_into_subdomain, + redirect_to_deactivation_notice, +) +from zproject.backends import ( + ExternalAuthResult, + ZulipLDAPAuthBackend, + ZulipLDAPExceptionNoMatchingLDAPUser, + any_social_backend_enabled, + email_auth_enabled, + email_belongs_to_ldap, + ldap_auth_enabled, + password_auth_enabled, +) -from confirmation import settings as confirmation_settings - -import logging -import smtplib - -import urllib def check_prereg_key_and_redirect(request: HttpRequest, confirmation_key: str) -> HttpResponse: confirmation = Confirmation.objects.filter(confirmation_key=confirmation_key).first() diff --git a/zerver/views/report.py b/zerver/views/report.py index de1d62d549532..3e665c58a94a4 100644 --- a/zerver/views/report.py +++ b/zerver/views/report.py @@ -1,25 +1,24 @@ # System documented in https://zulip.readthedocs.io/en/latest/subsystems/logging.html - +import logging +import subprocess from typing import Any, Dict, Optional from django.conf import settings from django.http import HttpRequest, HttpResponse from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_POST + from zerver.decorator import human_users_only from zerver.lib.bugdown import privacy_clean_markdown -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.response import json_success from zerver.lib.queue import queue_json_publish +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_success from zerver.lib.storage import static_path from zerver.lib.unminify import SourceMap from zerver.lib.utils import statsd, statsd_key from zerver.lib.validator import check_bool, check_dict, to_non_negative_int from zerver.models import UserProfile -import subprocess -import logging - js_source_map: Optional[SourceMap] = None # Read the source map information for decoding JavaScript backtraces. diff --git a/zerver/views/storage.py b/zerver/views/storage.py index 23e7214640d73..3772345cc0b9b 100644 --- a/zerver/views/storage.py +++ b/zerver/views/storage.py @@ -1,17 +1,19 @@ +from typing import Dict, List, Optional + from django.http import HttpRequest, HttpResponse + +from zerver.decorator import REQ, has_request_variables from zerver.lib.bot_storage import ( + StateError, get_bot_storage, - set_bot_storage, - remove_bot_storage, get_keys_in_bot_storage, - StateError, + remove_bot_storage, + set_bot_storage, ) -from zerver.decorator import has_request_variables, REQ -from zerver.lib.response import json_success, json_error +from zerver.lib.response import json_error, json_success from zerver.lib.validator import check_dict, check_list, check_string from zerver.models import UserProfile -from typing import Dict, List, Optional @has_request_variables def update_storage(request: HttpRequest, user_profile: UserProfile, diff --git a/zerver/views/streams.py b/zerver/views/streams.py index f8513a3372d8d..c8a36a24a4221 100644 --- a/zerver/views/streams.py +++ b/zerver/views/streams.py @@ -1,39 +1,79 @@ -from typing import Any, Optional, Tuple, List, Set, Iterable, Mapping, Callable, Dict, \ - Union +from collections import defaultdict +from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Set, Tuple, Union -from django.utils.translation import ugettext as _ +import ujson from django.conf import settings from django.db import transaction from django.http import HttpRequest, HttpResponse +from django.utils.translation import ugettext as _ -from zerver.lib.exceptions import JsonableError, ErrorCode +from zerver.decorator import ( + authenticated_json_post_view, + require_non_guest_user, + require_realm_admin, +) +from zerver.lib.actions import ( + bulk_add_subscriptions, + bulk_remove_subscriptions, + do_add_default_stream, + do_add_streams_to_default_stream_group, + do_change_default_stream_group_description, + do_change_default_stream_group_name, + do_change_stream_description, + do_change_stream_invite_only, + do_change_stream_post_policy, + do_change_subscription_property, + do_create_default_stream_group, + do_deactivate_stream, + do_delete_messages, + do_get_streams, + do_remove_default_stream, + do_remove_default_stream_group, + do_remove_streams_from_default_stream_group, + do_rename_stream, + do_send_messages, + gather_subscriptions, + get_subscriber_emails, + internal_prep_private_message, + internal_prep_stream_message, +) +from zerver.lib.exceptions import ErrorCode, JsonableError from zerver.lib.request import REQ, has_request_variables -from zerver.decorator import authenticated_json_post_view, \ - require_realm_admin, require_non_guest_user -from zerver.lib.actions import bulk_remove_subscriptions, \ - do_change_subscription_property, internal_prep_private_message, \ - internal_prep_stream_message, \ - gather_subscriptions, \ - bulk_add_subscriptions, do_send_messages, get_subscriber_emails, do_rename_stream, \ - do_deactivate_stream, do_change_stream_invite_only, do_add_default_stream, \ - do_change_stream_description, do_get_streams, \ - do_remove_default_stream, do_change_stream_post_policy, do_delete_messages, \ - do_create_default_stream_group, do_add_streams_to_default_stream_group, \ - do_remove_streams_from_default_stream_group, do_remove_default_stream_group, \ - do_change_default_stream_group_description, do_change_default_stream_group_name -from zerver.lib.response import json_success, json_error -from zerver.lib.streams import access_stream_by_id, access_stream_by_name, \ - check_stream_name, check_stream_name_available, filter_stream_authorization, \ - list_to_streams, access_stream_for_delete_or_update, access_default_stream_group_by_id +from zerver.lib.response import json_error, json_success +from zerver.lib.streams import ( + access_default_stream_group_by_id, + access_stream_by_id, + access_stream_by_name, + access_stream_for_delete_or_update, + check_stream_name, + check_stream_name_available, + filter_stream_authorization, + list_to_streams, +) from zerver.lib.topic import get_topic_history_for_stream, messages_for_topic -from zerver.lib.validator import check_string, check_int, check_list, check_dict, \ - check_bool, check_variable_type, check_capped_string, check_color, check_dict_only, \ - check_int_in, to_non_negative_int -from zerver.models import UserProfile, Stream, Realm, UserMessage, \ - get_system_bot, get_active_user, get_active_user_profile_by_id_in_realm +from zerver.lib.validator import ( + check_bool, + check_capped_string, + check_color, + check_dict, + check_dict_only, + check_int, + check_int_in, + check_list, + check_string, + check_variable_type, + to_non_negative_int, +) +from zerver.models import ( + Realm, + Stream, + UserMessage, + UserProfile, + get_active_user, + get_active_user_profile_by_id_in_realm, + get_system_bot, +) -from collections import defaultdict -import ujson class PrincipalError(JsonableError): code = ErrorCode.UNAUTHORIZED_PRINCIPAL diff --git a/zerver/views/submessage.py b/zerver/views/submessage.py index 014cb80b90bb2..0c0346949af80 100644 --- a/zerver/views/submessage.py +++ b/zerver/views/submessage.py @@ -1,21 +1,15 @@ import ujson - from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ -from zerver.decorator import ( - has_request_variables, - REQ, -) +from zerver.decorator import REQ, has_request_variables from zerver.lib.actions import do_add_submessage from zerver.lib.message import access_message +from zerver.lib.response import json_error, json_success from zerver.lib.validator import check_int -from zerver.lib.response import ( - json_error, - json_success, -) from zerver.models import UserProfile + @has_request_variables def process_submessage(request: HttpRequest, user_profile: UserProfile, diff --git a/zerver/views/thumbnail.py b/zerver/views/thumbnail.py index 2156d07720430..7d5bdf2ff5531 100644 --- a/zerver/views/thumbnail.py +++ b/zerver/views/thumbnail.py @@ -1,11 +1,14 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/thumbnailing.html +from typing import Optional + +from django.http import HttpRequest, HttpResponse, HttpResponseForbidden from django.shortcuts import redirect from django.utils.translation import ugettext as _ -from django.http import HttpRequest, HttpResponse, HttpResponseForbidden -from typing import Optional -from zerver.models import UserProfile, validate_attachment_request -from zerver.lib.request import has_request_variables, REQ + +from zerver.lib.request import REQ, has_request_variables from zerver.lib.thumbnail import generate_thumbnail_url +from zerver.models import UserProfile, validate_attachment_request + def validate_thumbnail_request(user_profile: UserProfile, path: str) -> Optional[bool]: # path here does not have a leading / as it is parsed from request hitting the diff --git a/zerver/views/tutorial.py b/zerver/views/tutorial.py index 5b647af7f3508..93ff426061172 100644 --- a/zerver/views/tutorial.py +++ b/zerver/views/tutorial.py @@ -1,11 +1,12 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import human_users_only -from zerver.lib.request import has_request_variables, REQ +from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.validator import check_string from zerver.models import UserProfile + @human_users_only @has_request_variables def set_tutorial_status(request: HttpRequest, user_profile: UserProfile, diff --git a/zerver/views/typing.py b/zerver/views/typing.py index 08f5ca1b01c41..1b2d143a71ca5 100644 --- a/zerver/views/typing.py +++ b/zerver/views/typing.py @@ -1,12 +1,14 @@ -from django.http import HttpRequest, HttpResponse from typing import List -from zerver.decorator import has_request_variables, REQ +from django.http import HttpRequest, HttpResponse + +from zerver.decorator import REQ, has_request_variables from zerver.lib.actions import check_send_typing_notification from zerver.lib.response import json_success from zerver.lib.validator import check_int, check_list from zerver.models import UserProfile + @has_request_variables def send_notification_backend( request: HttpRequest, diff --git a/zerver/views/unsubscribe.py b/zerver/views/unsubscribe.py index 4b5b56128124d..d22a2e819019d 100644 --- a/zerver/views/unsubscribe.py +++ b/zerver/views/unsubscribe.py @@ -1,13 +1,14 @@ +from typing import Callable + from django.http import HttpRequest, HttpResponse from django.shortcuts import render -from typing import Callable -from confirmation.models import Confirmation, get_object_from_key, \ - ConfirmationKeyException +from confirmation.models import Confirmation, ConfirmationKeyException, get_object_from_key +from zerver.context_processors import common_context from zerver.lib.actions import do_change_notification_settings from zerver.lib.send_email import clear_scheduled_emails -from zerver.models import UserProfile, ScheduledEmail -from zerver.context_processors import common_context +from zerver.models import ScheduledEmail, UserProfile + def process_unsubscribe(request: HttpRequest, confirmation_key: str, subscription_type: str, unsubscribe_function: Callable[[UserProfile], None]) -> HttpResponse: diff --git a/zerver/views/upload.py b/zerver/views/upload.py index 22ff8389b443d..eaa724e29b505 100644 --- a/zerver/views/upload.py +++ b/zerver/views/upload.py @@ -1,17 +1,24 @@ -from django.http import HttpRequest, HttpResponse, HttpResponseForbidden, \ - HttpResponseNotFound +from mimetypes import guess_type + +from django.conf import settings +from django.http import HttpRequest, HttpResponse, HttpResponseForbidden, HttpResponseNotFound from django.shortcuts import redirect from django.utils.cache import patch_cache_control from django.utils.translation import ugettext as _ +from django_sendfile import sendfile -from zerver.lib.response import json_success, json_error -from zerver.lib.upload import upload_message_image_from_request, get_local_file_path, \ - get_signed_upload_url, check_upload_within_quota, INLINE_MIME_TYPES, \ - generate_unauthed_file_access_url, get_local_file_path_id_from_token +from zerver.lib.response import json_error, json_success +from zerver.lib.upload import ( + INLINE_MIME_TYPES, + check_upload_within_quota, + generate_unauthed_file_access_url, + get_local_file_path, + get_local_file_path_id_from_token, + get_signed_upload_url, + upload_message_image_from_request, +) from zerver.models import UserProfile, validate_attachment_request -from django.conf import settings -from django_sendfile import sendfile -from mimetypes import guess_type + def serve_s3(request: HttpRequest, url_path: str, url_only: bool) -> HttpResponse: url = get_signed_upload_url(url_path) diff --git a/zerver/views/user_groups.py b/zerver/views/user_groups.py index 2aaddeda8705e..852823ee6327a 100644 --- a/zerver/views/user_groups.py +++ b/zerver/views/user_groups.py @@ -1,21 +1,31 @@ -from django.http import HttpResponse, HttpRequest -from django.utils.translation import ugettext as _ - from typing import List +from django.http import HttpRequest, HttpResponse +from django.utils.translation import ugettext as _ + from zerver.decorator import require_member_or_admin, require_user_group_edit_permission -from zerver.lib.actions import check_add_user_group, do_update_user_group_name, \ - do_update_user_group_description, bulk_add_members_to_user_group, \ - remove_members_from_user_group, check_delete_user_group +from zerver.lib.actions import ( + bulk_add_members_to_user_group, + check_add_user_group, + check_delete_user_group, + do_update_user_group_description, + do_update_user_group_name, + remove_members_from_user_group, +) from zerver.lib.exceptions import JsonableError -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.response import json_success, json_error +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_error, json_success +from zerver.lib.user_groups import ( + access_user_group_by_id, + get_memberships_of_users, + get_user_group_members, + user_groups_in_realm_serialized, +) from zerver.lib.users import user_ids_to_users -from zerver.lib.validator import check_list, check_int -from zerver.lib.user_groups import access_user_group_by_id, get_memberships_of_users, \ - get_user_group_members, user_groups_in_realm_serialized +from zerver.lib.validator import check_int, check_list from zerver.models import UserProfile -from zerver.views.streams import compose_views, FuncKwargPair +from zerver.views.streams import FuncKwargPair, compose_views + @require_user_group_edit_permission @has_request_variables diff --git a/zerver/views/user_settings.py b/zerver/views/user_settings.py index 86999d063a830..b19fe08ff3268 100644 --- a/zerver/views/user_settings.py +++ b/zerver/views/user_settings.py @@ -1,35 +1,47 @@ -from typing import Optional, Any, Dict +from typing import Any, Dict, Optional -from django.core.exceptions import ValidationError -from django.utils.translation import ugettext as _ from django.conf import settings from django.contrib.auth import authenticate, update_session_auth_hash +from django.core.exceptions import ValidationError from django.http import HttpRequest, HttpResponse from django.shortcuts import render +from django.utils.translation import ugettext as _ -from zerver.decorator import has_request_variables, \ - REQ, human_users_only -from zerver.lib.actions import do_change_password, do_change_notification_settings, \ - do_change_enter_sends, do_regenerate_api_key, do_change_avatar_fields, \ - do_set_user_display_setting, do_change_user_delivery_email, \ - do_start_email_change_process, check_change_full_name, \ - get_available_notification_sounds, validate_email_is_valid +from confirmation.models import ( + Confirmation, + ConfirmationKeyException, + get_object_from_key, + render_confirmation_key_error, +) +from zerver.decorator import REQ, has_request_variables, human_users_only +from zerver.lib.actions import ( + check_change_full_name, + do_change_avatar_fields, + do_change_enter_sends, + do_change_notification_settings, + do_change_password, + do_change_user_delivery_email, + do_regenerate_api_key, + do_set_user_display_setting, + do_start_email_change_process, + get_available_notification_sounds, + validate_email_is_valid, +) from zerver.lib.avatar import avatar_url -from zerver.lib.email_validation import get_realm_email_validator, \ - validate_email_not_already_in_realm -from zerver.lib.send_email import send_email, FromAddress +from zerver.lib.email_validation import ( + get_realm_email_validator, + validate_email_not_already_in_realm, +) from zerver.lib.i18n import get_available_language_codes -from zerver.lib.response import json_success, json_error -from zerver.lib.upload import upload_avatar_image -from zerver.lib.validator import check_bool, check_string, check_int, check_int_in, \ - check_string_in from zerver.lib.rate_limiter import RateLimited from zerver.lib.request import JsonableError +from zerver.lib.response import json_error, json_success +from zerver.lib.send_email import FromAddress, send_email from zerver.lib.timezone import get_all_timezones -from zerver.models import UserProfile, name_changes_disabled, avatar_changes_disabled -from confirmation.models import get_object_from_key, render_confirmation_key_error, \ - ConfirmationKeyException, Confirmation -from zproject.backends import email_belongs_to_ldap, check_password_strength +from zerver.lib.upload import upload_avatar_image +from zerver.lib.validator import check_bool, check_int, check_int_in, check_string, check_string_in +from zerver.models import UserProfile, avatar_changes_disabled, name_changes_disabled +from zproject.backends import check_password_strength, email_belongs_to_ldap AVATAR_CHANGES_DISABLED_ERROR = _("Avatar changes are disabled in this organization.") diff --git a/zerver/views/users.py b/zerver/views/users.py index 6e8af2d74737c..fd5e926f3a13c 100644 --- a/zerver/views/users.py +++ b/zerver/views/users.py @@ -1,44 +1,84 @@ -from typing import Union, Optional, Dict, Any, List +from typing import Any, Dict, List, Optional, Union +from django.conf import settings from django.http import HttpRequest, HttpResponse - -from django.utils.translation import ugettext as _ from django.shortcuts import redirect -from django.conf import settings +from django.utils.translation import ugettext as _ -from zerver.decorator import require_realm_admin, require_member_or_admin -from zerver.forms import CreateUserForm, PASSWORD_TOO_WEAK_ERROR -from zerver.lib.actions import do_change_avatar_fields, do_change_bot_owner, \ - do_change_user_role, do_change_default_all_public_streams, \ - do_change_default_events_register_stream, do_change_default_sending_stream, \ - do_create_user, do_deactivate_user, do_reactivate_user, do_regenerate_api_key, \ - check_change_full_name, notify_created_bot, do_update_outgoing_webhook_service, \ - do_update_bot_config_data, check_change_bot_full_name, \ - do_update_user_custom_profile_data_if_changed, check_remove_custom_profile_field_value +from zerver.decorator import require_member_or_admin, require_realm_admin +from zerver.forms import PASSWORD_TOO_WEAK_ERROR, CreateUserForm +from zerver.lib.actions import ( + check_change_bot_full_name, + check_change_full_name, + check_remove_custom_profile_field_value, + do_change_avatar_fields, + do_change_bot_owner, + do_change_default_all_public_streams, + do_change_default_events_register_stream, + do_change_default_sending_stream, + do_change_user_role, + do_create_user, + do_deactivate_user, + do_reactivate_user, + do_regenerate_api_key, + do_update_bot_config_data, + do_update_outgoing_webhook_service, + do_update_user_custom_profile_data_if_changed, + notify_created_bot, +) from zerver.lib.avatar import avatar_url, get_gravatar_url from zerver.lib.bot_config import set_bot_config from zerver.lib.email_validation import email_allowed_for_realm from zerver.lib.exceptions import CannotDeactivateLastUserError from zerver.lib.integrations import EMBEDDED_BOTS -from zerver.lib.request import has_request_variables, REQ +from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_error, json_success -from zerver.lib.streams import access_stream_by_name, access_stream_by_id, subscribed_to_stream +from zerver.lib.streams import access_stream_by_id, access_stream_by_name, subscribed_to_stream from zerver.lib.upload import upload_avatar_image -from zerver.lib.validator import check_bool, check_string, check_int, check_url, check_dict, check_list, \ - check_int_in from zerver.lib.url_encoding import add_query_arg_to_redirect_url -from zerver.lib.users import check_valid_bot_type, check_bot_creation_policy, \ - check_full_name, check_short_name, check_valid_interface_type, check_valid_bot_config, \ - access_bot_by_id, add_service, access_user_by_id, check_bot_name_available, \ - validate_user_custom_profile_data, get_raw_user_data, get_api_key +from zerver.lib.users import ( + access_bot_by_id, + access_user_by_id, + add_service, + check_bot_creation_policy, + check_bot_name_available, + check_full_name, + check_short_name, + check_valid_bot_config, + check_valid_bot_type, + check_valid_interface_type, + get_api_key, + get_raw_user_data, + validate_user_custom_profile_data, +) from zerver.lib.utils import generate_api_key -from zerver.models import UserProfile, Stream, Message, \ - get_user_by_delivery_email, Service, get_user_including_cross_realm, \ - DomainNotAllowedForRealmError, DisposableEmailError, get_user_profile_by_id_in_realm, \ - EmailContainsPlusError, get_user_by_id_in_realm_including_cross_realm, Realm, \ - InvalidFakeEmailDomain +from zerver.lib.validator import ( + check_bool, + check_dict, + check_int, + check_int_in, + check_list, + check_string, + check_url, +) +from zerver.models import ( + DisposableEmailError, + DomainNotAllowedForRealmError, + EmailContainsPlusError, + InvalidFakeEmailDomain, + Message, + Realm, + Service, + Stream, + UserProfile, + get_user_by_delivery_email, + get_user_by_id_in_realm_including_cross_realm, + get_user_including_cross_realm, + get_user_profile_by_id_in_realm, +) from zproject.backends import check_password_strength + def check_last_owner(user_profile: UserProfile) -> bool: owners = set(user_profile.realm.get_human_owner_users()) return user_profile.is_realm_owner and not user_profile.is_bot and len(owners) == 1 diff --git a/zerver/views/video_calls.py b/zerver/views/video_calls.py index a1c43f084b94a..f36373702f204 100644 --- a/zerver/views/video_calls.py +++ b/zerver/views/video_calls.py @@ -1,10 +1,11 @@ -from functools import partial import json +from functools import partial from typing import Dict from urllib.parse import urljoin +import requests from django.conf import settings -from django.http import HttpResponse, HttpRequest +from django.http import HttpRequest, HttpResponse from django.middleware import csrf from django.shortcuts import redirect, render from django.utils.crypto import constant_time_compare, salted_hmac @@ -13,7 +14,6 @@ from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_POST from oauthlib.oauth2 import OAuth2Error -import requests from requests_oauthlib import OAuth2Session from zerver.decorator import REQ, has_request_variables, zulip_login_required diff --git a/zerver/views/zephyr.py b/zerver/views/zephyr.py index 99db0796ae060..71c12ce588688 100644 --- a/zerver/views/zephyr.py +++ b/zerver/views/zephyr.py @@ -1,23 +1,22 @@ +import base64 +import logging +import re +import subprocess +from typing import Optional + +import ujson from django.conf import settings -from django.http import HttpResponse, HttpRequest +from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ + from zerver.decorator import authenticated_json_view from zerver.lib.ccache import make_ccache from zerver.lib.pysa import mark_sanitized -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.response import json_success, json_error +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_error, json_success from zerver.lib.users import get_api_key from zerver.models import UserProfile -import base64 -import logging -import re -import subprocess -import ujson - -from typing import Optional - - # Hack for mit.edu users whose Kerberos usernames don't match what they zephyr # as. The key is for Kerberos and the value is for zephyr. kerberos_alter_egos = { diff --git a/zerver/webhooks/alertmanager/view.py b/zerver/webhooks/alertmanager/view.py index 4156da5e727fe..11177d1008eeb 100644 --- a/zerver/webhooks/alertmanager/view.py +++ b/zerver/webhooks/alertmanager/view.py @@ -9,6 +9,7 @@ from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile + @api_key_only_webhook_view('AlertManager') @has_request_variables def api_alertmanager_webhook(request: HttpRequest, user_profile: UserProfile, diff --git a/zerver/webhooks/ansibletower/view.py b/zerver/webhooks/ansibletower/view.py index b41329c8f5659..77d0676d185f8 100644 --- a/zerver/webhooks/ansibletower/view.py +++ b/zerver/webhooks/ansibletower/view.py @@ -3,8 +3,7 @@ from django.http import HttpRequest, HttpResponse -from zerver.decorator import REQ, api_key_only_webhook_view, \ - has_request_variables +from zerver.decorator import REQ, api_key_only_webhook_view, has_request_variables from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/appveyor/view.py b/zerver/webhooks/appveyor/view.py index 9caadc9fe8de4..40914e1ef2afb 100644 --- a/zerver/webhooks/appveyor/view.py +++ b/zerver/webhooks/appveyor/view.py @@ -2,8 +2,7 @@ from django.http import HttpRequest, HttpResponse -from zerver.decorator import REQ, api_key_only_webhook_view, \ - has_request_variables +from zerver.decorator import REQ, api_key_only_webhook_view, has_request_variables from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/basecamp/view.py b/zerver/webhooks/basecamp/view.py index 8ba5412070211..74793d38267a6 100644 --- a/zerver/webhooks/basecamp/view.py +++ b/zerver/webhooks/basecamp/view.py @@ -7,8 +7,7 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile from .support_event import SUPPORT_EVENTS diff --git a/zerver/webhooks/beanstalk/tests.py b/zerver/webhooks/beanstalk/tests.py index 162ce7b708c0e..fe6f56e0220c0 100644 --- a/zerver/webhooks/beanstalk/tests.py +++ b/zerver/webhooks/beanstalk/tests.py @@ -1,5 +1,4 @@ from typing import Dict - from unittest.mock import MagicMock, patch from zerver.lib.test_classes import WebhookTestCase diff --git a/zerver/webhooks/beanstalk/view.py b/zerver/webhooks/beanstalk/view.py index 114642cdc397a..2e23e457882a9 100644 --- a/zerver/webhooks/beanstalk/view.py +++ b/zerver/webhooks/beanstalk/view.py @@ -12,8 +12,7 @@ from zerver.lib.types import ViewFuncT from zerver.lib.validator import check_dict from zerver.lib.webhooks.common import check_send_webhook_message -from zerver.lib.webhooks.git import TOPIC_WITH_BRANCH_TEMPLATE, \ - get_push_commits_event_message +from zerver.lib.webhooks.git import TOPIC_WITH_BRANCH_TEMPLATE, get_push_commits_event_message from zerver.models import UserProfile diff --git a/zerver/webhooks/bitbucket/tests.py b/zerver/webhooks/bitbucket/tests.py index b1979283adb32..61fe6cf6aca4e 100644 --- a/zerver/webhooks/bitbucket/tests.py +++ b/zerver/webhooks/bitbucket/tests.py @@ -1,5 +1,4 @@ from typing import Dict, Union - from unittest.mock import MagicMock, patch from zerver.lib.test_classes import WebhookTestCase diff --git a/zerver/webhooks/bitbucket/view.py b/zerver/webhooks/bitbucket/view.py index 583ef1ce7134c..e2d9579addc59 100644 --- a/zerver/webhooks/bitbucket/view.py +++ b/zerver/webhooks/bitbucket/view.py @@ -7,8 +7,7 @@ from zerver.lib.response import json_success from zerver.lib.validator import check_dict from zerver.lib.webhooks.common import check_send_webhook_message -from zerver.lib.webhooks.git import TOPIC_WITH_BRANCH_TEMPLATE, \ - get_push_commits_event_message +from zerver.lib.webhooks.git import TOPIC_WITH_BRANCH_TEMPLATE, get_push_commits_event_message from zerver.models import UserProfile diff --git a/zerver/webhooks/bitbucket2/view.py b/zerver/webhooks/bitbucket2/view.py index 1333b62f77f16..35bae5d0bac33 100644 --- a/zerver/webhooks/bitbucket2/view.py +++ b/zerver/webhooks/bitbucket2/view.py @@ -10,13 +10,22 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message, validate_extract_webhook_http_header -from zerver.lib.webhooks.git import TOPIC_WITH_BRANCH_TEMPLATE, \ - TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, get_commits_comment_action_message, \ - get_force_push_commits_event_message, get_issue_event_message, \ - get_pull_request_event_message, get_push_commits_event_message, \ - get_push_tag_event_message, get_remove_branch_event_message +from zerver.lib.webhooks.common import ( + UnexpectedWebhookEventType, + check_send_webhook_message, + validate_extract_webhook_http_header, +) +from zerver.lib.webhooks.git import ( + TOPIC_WITH_BRANCH_TEMPLATE, + TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, + get_commits_comment_action_message, + get_force_push_commits_event_message, + get_issue_event_message, + get_pull_request_event_message, + get_push_commits_event_message, + get_push_tag_event_message, + get_remove_branch_event_message, +) from zerver.models import UserProfile BITBUCKET_TOPIC_TEMPLATE = '{repository_name}' diff --git a/zerver/webhooks/bitbucket3/view.py b/zerver/webhooks/bitbucket3/view.py index 29dc805f03d88..cb6bae92d7391 100644 --- a/zerver/webhooks/bitbucket3/view.py +++ b/zerver/webhooks/bitbucket3/view.py @@ -8,16 +8,23 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message -from zerver.lib.webhooks.git import CONTENT_MESSAGE_TEMPLATE, \ - TOPIC_WITH_BRANCH_TEMPLATE, TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, \ - get_commits_comment_action_message, get_create_branch_event_message, \ - get_pull_request_event_message, get_push_tag_event_message, \ - get_remove_branch_event_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message +from zerver.lib.webhooks.git import ( + CONTENT_MESSAGE_TEMPLATE, + TOPIC_WITH_BRANCH_TEMPLATE, + TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, + get_commits_comment_action_message, + get_create_branch_event_message, + get_pull_request_event_message, + get_push_tag_event_message, + get_remove_branch_event_message, +) from zerver.models import UserProfile -from zerver.webhooks.bitbucket2.view import BITBUCKET_FORK_BODY, \ - BITBUCKET_REPO_UPDATED_CHANGED, BITBUCKET_TOPIC_TEMPLATE +from zerver.webhooks.bitbucket2.view import ( + BITBUCKET_FORK_BODY, + BITBUCKET_REPO_UPDATED_CHANGED, + BITBUCKET_TOPIC_TEMPLATE, +) BRANCH_UPDATED_MESSAGE_TEMPLATE = "{user_name} pushed to branch {branch_name}. Head is now {head}." PULL_REQUEST_MARKED_AS_NEEDS_WORK_TEMPLATE = "{user_name} marked [PR #{number}]({url}) as \"needs work\"." diff --git a/zerver/webhooks/clubhouse/tests.py b/zerver/webhooks/clubhouse/tests.py index 72f802c430a0e..76494504eab58 100644 --- a/zerver/webhooks/clubhouse/tests.py +++ b/zerver/webhooks/clubhouse/tests.py @@ -1,5 +1,4 @@ import json - from unittest.mock import MagicMock, patch from zerver.lib.test_classes import WebhookTestCase diff --git a/zerver/webhooks/clubhouse/view.py b/zerver/webhooks/clubhouse/view.py index 68628753e9437..2987cddd720f2 100644 --- a/zerver/webhooks/clubhouse/view.py +++ b/zerver/webhooks/clubhouse/view.py @@ -6,8 +6,7 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile EPIC_NAME_TEMPLATE = "**{name}**" diff --git a/zerver/webhooks/flock/view.py b/zerver/webhooks/flock/view.py index 99f2232303f0a..dd8938387d2d4 100644 --- a/zerver/webhooks/flock/view.py +++ b/zerver/webhooks/flock/view.py @@ -3,8 +3,7 @@ from django.http import HttpRequest, HttpResponse -from zerver.decorator import REQ, api_key_only_webhook_view, \ - has_request_variables +from zerver.decorator import REQ, api_key_only_webhook_view, has_request_variables from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/github/view.py b/zerver/webhooks/github/view.py index 7fd1510f04042..1e677e0c679c4 100644 --- a/zerver/webhooks/github/view.py +++ b/zerver/webhooks/github/view.py @@ -8,15 +8,24 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message, get_http_headers_from_filename, \ - validate_extract_webhook_http_header -from zerver.lib.webhooks.git import CONTENT_MESSAGE_TEMPLATE, \ - TOPIC_WITH_BRANCH_TEMPLATE, TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, \ - get_commits_comment_action_message, get_issue_event_message, \ - get_pull_request_event_message, get_push_commits_event_message, \ - get_push_tag_event_message, get_setup_webhook_message, \ - get_release_event_message +from zerver.lib.webhooks.common import ( + UnexpectedWebhookEventType, + check_send_webhook_message, + get_http_headers_from_filename, + validate_extract_webhook_http_header, +) +from zerver.lib.webhooks.git import ( + CONTENT_MESSAGE_TEMPLATE, + TOPIC_WITH_BRANCH_TEMPLATE, + TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, + get_commits_comment_action_message, + get_issue_event_message, + get_pull_request_event_message, + get_push_commits_event_message, + get_push_tag_event_message, + get_release_event_message, + get_setup_webhook_message, +) from zerver.models import UserProfile fixture_to_headers = get_http_headers_from_filename("HTTP_X_GITHUB_EVENT") diff --git a/zerver/webhooks/gitlab/view.py b/zerver/webhooks/gitlab/view.py index 80a6178070295..6e10e2e19c0e7 100644 --- a/zerver/webhooks/gitlab/view.py +++ b/zerver/webhooks/gitlab/view.py @@ -8,13 +8,21 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message, validate_extract_webhook_http_header -from zerver.lib.webhooks.git import EMPTY_SHA, \ - TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, get_commits_comment_action_message, \ - get_issue_event_message, get_pull_request_event_message, \ - get_push_commits_event_message, get_push_tag_event_message, \ - get_remove_branch_event_message +from zerver.lib.webhooks.common import ( + UnexpectedWebhookEventType, + check_send_webhook_message, + validate_extract_webhook_http_header, +) +from zerver.lib.webhooks.git import ( + EMPTY_SHA, + TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, + get_commits_comment_action_message, + get_issue_event_message, + get_pull_request_event_message, + get_push_commits_event_message, + get_push_tag_event_message, + get_remove_branch_event_message, +) from zerver.models import UserProfile diff --git a/zerver/webhooks/gogs/view.py b/zerver/webhooks/gogs/view.py index e43f2b75adf8e..2f4a46300ff67 100644 --- a/zerver/webhooks/gogs/view.py +++ b/zerver/webhooks/gogs/view.py @@ -6,14 +6,22 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message, get_http_headers_from_filename, \ - validate_extract_webhook_http_header -from zerver.lib.webhooks.git import TOPIC_WITH_BRANCH_TEMPLATE, \ - TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, TOPIC_WITH_RELEASE_TEMPLATE, \ - get_create_branch_event_message, \ - get_issue_event_message, get_pull_request_event_message, \ - get_push_commits_event_message, get_release_event_message +from zerver.lib.webhooks.common import ( + UnexpectedWebhookEventType, + check_send_webhook_message, + get_http_headers_from_filename, + validate_extract_webhook_http_header, +) +from zerver.lib.webhooks.git import ( + TOPIC_WITH_BRANCH_TEMPLATE, + TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, + TOPIC_WITH_RELEASE_TEMPLATE, + get_create_branch_event_message, + get_issue_event_message, + get_pull_request_event_message, + get_push_commits_event_message, + get_release_event_message, +) from zerver.models import UserProfile fixture_to_headers = get_http_headers_from_filename("HTTP_X_GOGS_EVENT") diff --git a/zerver/webhooks/gosquared/view.py b/zerver/webhooks/gosquared/view.py index ec6aacd5a3588..0c9affba7deda 100644 --- a/zerver/webhooks/gosquared/view.py +++ b/zerver/webhooks/gosquared/view.py @@ -5,8 +5,7 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile TRAFFIC_SPIKE_TEMPLATE = '[{website_name}]({website_url}) has {user_num} visitors online.' diff --git a/zerver/webhooks/grafana/view.py b/zerver/webhooks/grafana/view.py index 9051d7aaecc79..ebd70a8b66890 100644 --- a/zerver/webhooks/grafana/view.py +++ b/zerver/webhooks/grafana/view.py @@ -3,9 +3,9 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view -from zerver.lib.webhooks.common import check_send_webhook_message from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success +from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile GRAFANA_TOPIC_TEMPLATE = '{alert_title}' diff --git a/zerver/webhooks/groove/view.py b/zerver/webhooks/groove/view.py index 76b508056feb7..038003bdfae2f 100644 --- a/zerver/webhooks/groove/view.py +++ b/zerver/webhooks/groove/view.py @@ -7,9 +7,12 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message, get_http_headers_from_filename, \ - validate_extract_webhook_http_header +from zerver.lib.webhooks.common import ( + UnexpectedWebhookEventType, + check_send_webhook_message, + get_http_headers_from_filename, + validate_extract_webhook_http_header, +) from zerver.models import UserProfile TICKET_STARTED_TEMPLATE = """ diff --git a/zerver/webhooks/harbor/view.py b/zerver/webhooks/harbor/view.py index 64a6846cb778a..3dd7a35c054b1 100644 --- a/zerver/webhooks/harbor/view.py +++ b/zerver/webhooks/harbor/view.py @@ -7,8 +7,7 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import Realm, UserProfile IGNORED_EVENTS = [ diff --git a/zerver/webhooks/hellosign/tests.py b/zerver/webhooks/hellosign/tests.py index 2cadf71664531..79738fddac49e 100644 --- a/zerver/webhooks/hellosign/tests.py +++ b/zerver/webhooks/hellosign/tests.py @@ -1,6 +1,8 @@ -from zerver.lib.test_classes import WebhookTestCase from typing import Dict +from zerver.lib.test_classes import WebhookTestCase + + class HelloSignHookTests(WebhookTestCase): STREAM_NAME = 'hellosign' URL_TEMPLATE = "/api/v1/external/hellosign?stream={stream}&api_key={api_key}" diff --git a/zerver/webhooks/hellosign/view.py b/zerver/webhooks/hellosign/view.py index 6dd775af057ae..27018dd6b393e 100644 --- a/zerver/webhooks/hellosign/view.py +++ b/zerver/webhooks/hellosign/view.py @@ -1,5 +1,6 @@ from typing import Any, Dict, List +import ujson from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view @@ -7,7 +8,6 @@ from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile -import ujson IS_AWAITING_SIGNATURE = "is awaiting the signature of {awaiting_recipients}" WAS_JUST_SIGNED_BY = "was just signed by {signed_recipients}" diff --git a/zerver/webhooks/insping/view.py b/zerver/webhooks/insping/view.py index 152de3ad44d10..3acea44451e96 100644 --- a/zerver/webhooks/insping/view.py +++ b/zerver/webhooks/insping/view.py @@ -3,8 +3,7 @@ from django.http import HttpRequest, HttpResponse -from zerver.decorator import REQ, api_key_only_webhook_view, \ - has_request_variables +from zerver.decorator import REQ, api_key_only_webhook_view, has_request_variables from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/intercom/view.py b/zerver/webhooks/intercom/view.py index 35960af125625..fcee857de7064 100644 --- a/zerver/webhooks/intercom/view.py +++ b/zerver/webhooks/intercom/view.py @@ -7,8 +7,7 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile COMPANY_CREATED = """ diff --git a/zerver/webhooks/jira/view.py b/zerver/webhooks/jira/view.py index 05f8c6aef7dd8..ab095687df762 100644 --- a/zerver/webhooks/jira/view.py +++ b/zerver/webhooks/jira/view.py @@ -9,8 +9,7 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import Realm, UserProfile, get_user_by_delivery_email IGNORED_EVENTS = [ diff --git a/zerver/webhooks/netlify/view.py b/zerver/webhooks/netlify/view.py index 1f06303fca8de..1c256b1dfedf2 100644 --- a/zerver/webhooks/netlify/view.py +++ b/zerver/webhooks/netlify/view.py @@ -5,9 +5,12 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message, get_http_headers_from_filename, \ - validate_extract_webhook_http_header +from zerver.lib.webhooks.common import ( + UnexpectedWebhookEventType, + check_send_webhook_message, + get_http_headers_from_filename, + validate_extract_webhook_http_header, +) from zerver.models import UserProfile EVENTS = ['deploy_failed', 'deploy_locked', 'deploy_unlocked', 'deploy_building', 'deploy_created'] diff --git a/zerver/webhooks/newrelic/view.py b/zerver/webhooks/newrelic/view.py index 7848c811abe3f..8ef9bf0f2ee71 100644 --- a/zerver/webhooks/newrelic/view.py +++ b/zerver/webhooks/newrelic/view.py @@ -7,8 +7,7 @@ from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.validator import check_dict -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile ALERT_TEMPLATE = "{long_description} ([view alert]({alert_url}))." diff --git a/zerver/webhooks/pagerduty/view.py b/zerver/webhooks/pagerduty/view.py index 69576c38df762..9267bb9811867 100644 --- a/zerver/webhooks/pagerduty/view.py +++ b/zerver/webhooks/pagerduty/view.py @@ -6,8 +6,7 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile PAGER_DUTY_EVENT_NAMES = { diff --git a/zerver/webhooks/pingdom/view.py b/zerver/webhooks/pingdom/view.py index 1cb1ec5aedec7..16ac895043752 100644 --- a/zerver/webhooks/pingdom/view.py +++ b/zerver/webhooks/pingdom/view.py @@ -6,8 +6,7 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile PINGDOM_TOPIC_TEMPLATE = '{name} status.' diff --git a/zerver/webhooks/pivotal/view.py b/zerver/webhooks/pivotal/view.py index 3a56f4b2dd2c9..6fd45a82752a0 100644 --- a/zerver/webhooks/pivotal/view.py +++ b/zerver/webhooks/pivotal/view.py @@ -10,8 +10,7 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import has_request_variables from zerver.lib.response import json_error, json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/raygun/view.py b/zerver/webhooks/raygun/view.py index cace93e5bbe45..497c9c9e27a3d 100644 --- a/zerver/webhooks/raygun/view.py +++ b/zerver/webhooks/raygun/view.py @@ -6,8 +6,7 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/reviewboard/view.py b/zerver/webhooks/reviewboard/view.py index e89d2122b1f6f..e5e024d3ef4fc 100644 --- a/zerver/webhooks/reviewboard/view.py +++ b/zerver/webhooks/reviewboard/view.py @@ -5,9 +5,12 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message, get_http_headers_from_filename, \ - validate_extract_webhook_http_header +from zerver.lib.webhooks.common import ( + UnexpectedWebhookEventType, + check_send_webhook_message, + get_http_headers_from_filename, + validate_extract_webhook_http_header, +) from zerver.models import UserProfile REVIEW_REQUEST_PUBLISHED = """ diff --git a/zerver/webhooks/semaphore/tests.py b/zerver/webhooks/semaphore/tests.py index bcb487f17abd2..83356b5601c46 100644 --- a/zerver/webhooks/semaphore/tests.py +++ b/zerver/webhooks/semaphore/tests.py @@ -1,5 +1,7 @@ -import ujson from unittest.mock import patch + +import ujson + from zerver.lib.test_classes import WebhookTestCase diff --git a/zerver/webhooks/semaphore/view.py b/zerver/webhooks/semaphore/view.py index 77b0979a86780..4fe0fc3242a8b 100644 --- a/zerver/webhooks/semaphore/view.py +++ b/zerver/webhooks/semaphore/view.py @@ -1,6 +1,5 @@ # Webhooks for external integrations. -from typing import Any, Dict, Tuple, Optional - +from typing import Any, Dict, Optional, Tuple from urllib.parse import urlparse from django.http import HttpRequest, HttpResponse diff --git a/zerver/webhooks/sentry/view.py b/zerver/webhooks/sentry/view.py index 9d2aa7977c75b..4e42b6c31c119 100644 --- a/zerver/webhooks/sentry/view.py +++ b/zerver/webhooks/sentry/view.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Tuple, Optional +from typing import Any, Dict, List, Optional, Tuple from django.http import HttpRequest, HttpResponse @@ -8,7 +8,6 @@ from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile - DEPRECATED_EXCEPTION_MESSAGE_TEMPLATE = """ New [issue]({url}) (level: {level}): diff --git a/zerver/webhooks/slack_incoming/view.py b/zerver/webhooks/slack_incoming/view.py index 418953c657363..67eb9d9003207 100644 --- a/zerver/webhooks/slack_incoming/view.py +++ b/zerver/webhooks/slack_incoming/view.py @@ -1,17 +1,18 @@ # Webhooks for external integrations. +import re from typing import Any, Dict, Optional +import ujson from django.http import HttpRequest, HttpResponse +from django.utils.translation import ugettext as _ from zerver.decorator import api_key_only_webhook_view +from zerver.lib.exceptions import InvalidJSONError from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message -from zerver.lib.exceptions import InvalidJSONError -from django.utils.translation import ugettext as _ from zerver.models import UserProfile -import re -import ujson + @api_key_only_webhook_view('SlackIncoming') @has_request_variables diff --git a/zerver/webhooks/statuspage/view.py b/zerver/webhooks/statuspage/view.py index e5fa6b3f3b466..cf8e820b3185b 100644 --- a/zerver/webhooks/statuspage/view.py +++ b/zerver/webhooks/statuspage/view.py @@ -3,8 +3,7 @@ from django.http import HttpRequest, HttpResponse -from zerver.decorator import REQ, api_key_only_webhook_view, \ - has_request_variables +from zerver.decorator import REQ, api_key_only_webhook_view, has_request_variables from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/stripe/view.py b/zerver/webhooks/stripe/view.py index 57d542f748809..4b2ed311b9f52 100644 --- a/zerver/webhooks/stripe/view.py +++ b/zerver/webhooks/stripe/view.py @@ -8,8 +8,7 @@ from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.timestamp import timestamp_to_datetime -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/teamcity/tests.py b/zerver/webhooks/teamcity/tests.py index 268c90d078dd7..db3e8acee73ac 100644 --- a/zerver/webhooks/teamcity/tests.py +++ b/zerver/webhooks/teamcity/tests.py @@ -3,8 +3,7 @@ from zerver.lib.send_email import FromAddress from zerver.lib.test_classes import WebhookTestCase from zerver.models import Recipient, get_realm, get_user_by_delivery_email -from zerver.webhooks.teamcity.view import \ - MISCONFIGURED_PAYLOAD_TYPE_ERROR_MESSAGE +from zerver.webhooks.teamcity.view import MISCONFIGURED_PAYLOAD_TYPE_ERROR_MESSAGE class TeamcityHookTests(WebhookTestCase): diff --git a/zerver/webhooks/teamcity/view.py b/zerver/webhooks/teamcity/view.py index 2dc4dcecf718f..b83cfb0c826a6 100644 --- a/zerver/webhooks/teamcity/view.py +++ b/zerver/webhooks/teamcity/view.py @@ -6,8 +6,10 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view -from zerver.lib.actions import check_send_private_message, \ - send_rate_limited_pm_notification_to_bot_owner +from zerver.lib.actions import ( + check_send_private_message, + send_rate_limited_pm_notification_to_bot_owner, +) from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.send_email import FromAddress diff --git a/zerver/webhooks/transifex/view.py b/zerver/webhooks/transifex/view.py index 8614807fa09e6..7f22564963678 100644 --- a/zerver/webhooks/transifex/view.py +++ b/zerver/webhooks/transifex/view.py @@ -7,8 +7,7 @@ from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.validator import check_int -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/trello/view/__init__.py b/zerver/webhooks/trello/view/__init__.py index c45538303760a..e6a01d0c3c4bc 100644 --- a/zerver/webhooks/trello/view/__init__.py +++ b/zerver/webhooks/trello/view/__init__.py @@ -4,17 +4,14 @@ import ujson from django.http import HttpRequest, HttpResponse -from zerver.decorator import api_key_only_webhook_view, \ - return_success_on_head_request +from zerver.decorator import api_key_only_webhook_view, return_success_on_head_request from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile from .board_actions import SUPPORTED_BOARD_ACTIONS, process_board_action -from .card_actions import IGNORED_CARD_ACTIONS, SUPPORTED_CARD_ACTIONS, \ - process_card_action +from .card_actions import IGNORED_CARD_ACTIONS, SUPPORTED_CARD_ACTIONS, process_card_action @api_key_only_webhook_view('Trello') diff --git a/zerver/webhooks/updown/view.py b/zerver/webhooks/updown/view.py index 8151518822a67..d0c49b2eca3a5 100644 --- a/zerver/webhooks/updown/view.py +++ b/zerver/webhooks/updown/view.py @@ -7,8 +7,7 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile TOPIC_TEMPLATE = "{service_url}" diff --git a/zerver/webhooks/zabbix/view.py b/zerver/webhooks/zabbix/view.py index 846dcc4bbdd1b..5a59bd7c82bfd 100644 --- a/zerver/webhooks/zabbix/view.py +++ b/zerver/webhooks/zabbix/view.py @@ -3,8 +3,7 @@ from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ -from zerver.decorator import REQ, api_key_only_webhook_view, \ - has_request_variables +from zerver.decorator import REQ, api_key_only_webhook_view, has_request_variables from zerver.lib.actions import send_rate_limited_pm_notification_to_bot_owner from zerver.lib.response import json_error, json_success from zerver.lib.send_email import FromAddress diff --git a/zerver/worker/queue_processors.py b/zerver/worker/queue_processors.py index 1aa7cb202fe9c..b647706469fb5 100644 --- a/zerver/worker/queue_processors.py +++ b/zerver/worker/queue_processors.py @@ -1,61 +1,97 @@ # Documented in https://zulip.readthedocs.io/en/latest/subsystems/queuing.html -from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, List, Mapping, MutableSequence, Optional, cast, Tuple, TypeVar, Type - import copy +import datetime +import email +import logging +import os import signal +import smtplib +import socket import tempfile +import time +import urllib +from abc import ABC, abstractmethod +from collections import defaultdict, deque from functools import wraps from threading import Timer +from typing import ( + Any, + Callable, + Dict, + List, + Mapping, + MutableSequence, + Optional, + Tuple, + Type, + TypeVar, + cast, +) -import smtplib -import socket - +import requests +import ujson from django.conf import settings from django.db import connection from django.utils.timezone import now as timezone_now -from zerver.models import \ - get_client, get_system_bot, PreregistrationUser, \ - get_user_profile_by_id, Message, Realm, UserMessage, UserProfile, \ - Client, flush_per_request_caches +from zulip_bots.lib import ExternalBotHandler, extract_query_without_mention + +from zerver.context_processors import common_context +from zerver.lib.actions import ( + do_mark_stream_messages_as_read, + do_send_confirmation_email, + do_update_embedded_data, + do_update_user_activity, + do_update_user_activity_interval, + do_update_user_presence, + internal_send_private_message, + notify_realm_export, + render_incoming_message, +) +from zerver.lib.bot_lib import EmbeddedBotHandler, EmbeddedBotQuitException, get_bot_handler from zerver.lib.context_managers import lockfile -from zerver.lib.error_notify import do_report_error -from zerver.lib.queue import SimpleQueueClient, retry_event -from zerver.lib.timestamp import timestamp_to_datetime -from zerver.lib.email_notifications import handle_missedmessage_emails -from zerver.lib.push_notifications import handle_push_notification, handle_remove_push_notification, \ - initialize_push_notifications, clear_push_device_tokens -from zerver.lib.actions import do_send_confirmation_email, \ - do_update_user_activity, do_update_user_activity_interval, do_update_user_presence, \ - internal_send_private_message, notify_realm_export, \ - render_incoming_message, do_update_embedded_data, do_mark_stream_messages_as_read -from zerver.lib.url_preview import preview as url_preview -from zerver.lib.digest import handle_digest_email -from zerver.lib.send_email import send_future_email, send_email_from_dict, \ - FromAddress, EmailNotDeliveredException, handle_send_email_format_changes -from zerver.lib.email_mirror import process_message as mirror_email, rate_limit_mirror_by_realm, \ - is_missed_message_address, decode_stream_email_address -from zerver.lib.streams import access_stream_by_id from zerver.lib.db import reset_queries -from zerver.context_processors import common_context -from zerver.lib.outgoing_webhook import do_rest_call, get_outgoing_webhook_service_handler -from zerver.models import get_bot_services, RealmAuditLog -from zulip_bots.lib import ExternalBotHandler, extract_query_without_mention -from zerver.lib.bot_lib import EmbeddedBotHandler, get_bot_handler, EmbeddedBotQuitException +from zerver.lib.digest import handle_digest_email +from zerver.lib.email_mirror import decode_stream_email_address, is_missed_message_address +from zerver.lib.email_mirror import process_message as mirror_email +from zerver.lib.email_mirror import rate_limit_mirror_by_realm +from zerver.lib.email_notifications import handle_missedmessage_emails +from zerver.lib.error_notify import do_report_error from zerver.lib.exceptions import RateLimited from zerver.lib.export import export_realm_wrapper -from zerver.lib.remote_server import PushNotificationBouncerRetryLaterError +from zerver.lib.outgoing_webhook import do_rest_call, get_outgoing_webhook_service_handler +from zerver.lib.push_notifications import ( + clear_push_device_tokens, + handle_push_notification, + handle_remove_push_notification, + initialize_push_notifications, +) from zerver.lib.pysa import mark_sanitized - -import os -import ujson -from collections import defaultdict, deque -import email -import time -import datetime -import logging -import requests -import urllib +from zerver.lib.queue import SimpleQueueClient, retry_event +from zerver.lib.remote_server import PushNotificationBouncerRetryLaterError +from zerver.lib.send_email import ( + EmailNotDeliveredException, + FromAddress, + handle_send_email_format_changes, + send_email_from_dict, + send_future_email, +) +from zerver.lib.streams import access_stream_by_id +from zerver.lib.timestamp import timestamp_to_datetime +from zerver.lib.url_preview import preview as url_preview +from zerver.models import ( + Client, + Message, + PreregistrationUser, + Realm, + RealmAuditLog, + UserMessage, + UserProfile, + flush_per_request_caches, + get_bot_services, + get_client, + get_system_bot, + get_user_profile_by_id, +) logger = logging.getLogger(__name__) diff --git a/zilencer/forms.py b/zilencer/forms.py index 4a2352e749b00..b6f90c3832cef 100644 --- a/zilencer/forms.py +++ b/zilencer/forms.py @@ -1,5 +1,6 @@ from django import forms + class EnterpriseToSForm(forms.Form): full_name = forms.CharField(max_length=100) company = forms.CharField(max_length=100) diff --git a/zilencer/management/commands/add_mock_conversation.py b/zilencer/management/commands/add_mock_conversation.py index 69f40877e22e9..ae50776598c01 100644 --- a/zilencer/management/commands/add_mock_conversation.py +++ b/zilencer/management/commands/add_mock_conversation.py @@ -2,9 +2,15 @@ from django.core.management.base import BaseCommand -from zerver.lib.actions import bulk_add_subscriptions, do_add_reaction, \ - do_change_avatar_fields, do_create_user, do_send_messages, ensure_stream, \ - internal_prep_stream_message +from zerver.lib.actions import ( + bulk_add_subscriptions, + do_add_reaction, + do_change_avatar_fields, + do_create_user, + do_send_messages, + ensure_stream, + internal_prep_stream_message, +) from zerver.lib.emoji import emoji_name_to_emoji_code from zerver.lib.upload import upload_avatar_image from zerver.models import Message, UserProfile, get_realm diff --git a/zilencer/management/commands/add_new_realm.py b/zilencer/management/commands/add_new_realm.py index a78723114d768..170952ce561a2 100644 --- a/zilencer/management/commands/add_new_realm.py +++ b/zilencer/management/commands/add_new_realm.py @@ -1,7 +1,6 @@ from typing import Any -from zerver.lib.actions import bulk_add_subscriptions, do_create_realm, \ - do_create_user +from zerver.lib.actions import bulk_add_subscriptions, do_create_realm, do_create_user from zerver.lib.management import ZulipBaseCommand from zerver.lib.onboarding import send_initial_realm_messages from zerver.models import Realm, UserProfile diff --git a/zilencer/management/commands/populate_db.py b/zilencer/management/commands/populate_db.py index 86c3421ad84c5..0676eaa773e03 100644 --- a/zilencer/management/commands/populate_db.py +++ b/zilencer/management/commands/populate_db.py @@ -1,11 +1,11 @@ import itertools import os import random -from typing import Any, Callable, Dict, List, \ - Mapping, Sequence, Tuple +from datetime import datetime +from typing import Any, Callable, Dict, List, Mapping, Sequence, Tuple +import pylibmc import ujson -from datetime import datetime from django.conf import settings from django.contrib.sessions.models import Session from django.core.management import call_command @@ -13,11 +13,17 @@ from django.db.models import F, Max from django.utils.timezone import now as timezone_now from django.utils.timezone import timedelta as timezone_timedelta -import pylibmc -from zerver.lib.actions import STREAM_ASSIGNMENT_COLORS, check_add_realm_emoji, \ - do_change_user_role, do_send_messages, do_update_user_custom_profile_data_if_changed, \ - try_add_realm_custom_profile_field, try_add_realm_default_custom_profile_field +from scripts.lib.zulip_tools import get_or_create_dev_uuid_var_path +from zerver.lib.actions import ( + STREAM_ASSIGNMENT_COLORS, + check_add_realm_emoji, + do_change_user_role, + do_send_messages, + do_update_user_custom_profile_data_if_changed, + try_add_realm_custom_profile_field, + try_add_realm_default_custom_profile_field, +) from zerver.lib.bulk_create import bulk_create_streams from zerver.lib.cache import cache_set from zerver.lib.generate_test_data import create_test_data, generate_topics @@ -25,18 +31,35 @@ from zerver.lib.push_notifications import logger as push_notifications_logger from zerver.lib.server_initialization import create_internal_realm, create_users from zerver.lib.storage import static_path -from zerver.lib.users import add_service +from zerver.lib.types import ProfileFieldData from zerver.lib.url_preview.preview import CACHE_NAME as PREVIEW_CACHE_NAME from zerver.lib.user_groups import create_user_group +from zerver.lib.users import add_service from zerver.lib.utils import generate_api_key -from zerver.models import CustomProfileField, DefaultStream, Message, Realm, RealmAuditLog, \ - RealmDomain, Recipient, Service, Stream, Subscription, \ - UserMessage, UserPresence, UserProfile, Huddle, Client, \ - get_client, get_huddle, get_realm, get_stream, \ - get_user, get_user_by_delivery_email, get_user_profile_by_id -from zerver.lib.types import ProfileFieldData - -from scripts.lib.zulip_tools import get_or_create_dev_uuid_var_path +from zerver.models import ( + Client, + CustomProfileField, + DefaultStream, + Huddle, + Message, + Realm, + RealmAuditLog, + RealmDomain, + Recipient, + Service, + Stream, + Subscription, + UserMessage, + UserPresence, + UserProfile, + get_client, + get_huddle, + get_realm, + get_stream, + get_user, + get_user_by_delivery_email, + get_user_profile_by_id, +) settings.TORNADO_SERVER = None # Disable using memcached caches to avoid 'unsupported pickle diff --git a/zilencer/models.py b/zilencer/models.py index 91f93606cbf67..34481bbeb5328 100644 --- a/zilencer/models.py +++ b/zilencer/models.py @@ -2,8 +2,9 @@ from django.db import models -from zerver.models import AbstractPushDeviceToken, AbstractRealmAuditLog from analytics.models import BaseCount +from zerver.models import AbstractPushDeviceToken, AbstractRealmAuditLog + def get_remote_server_by_uuid(uuid: str) -> 'RemoteZulipServer': return RemoteZulipServer.objects.get(uuid=uuid) diff --git a/zilencer/views.py b/zilencer/views.py index fc137a41491ab..ad659c4f1c5bf 100644 --- a/zilencer/views.py +++ b/zilencer/views.py @@ -1,29 +1,46 @@ -from typing import Any, Dict, List, Optional, Union import datetime import logging +from typing import Any, Dict, List, Optional, Union from django.core.exceptions import ValidationError -from django.core.validators import validate_email, URLValidator +from django.core.validators import URLValidator, validate_email from django.db import IntegrityError, transaction from django.http import HttpRequest, HttpResponse from django.utils import timezone -from django.utils.translation import ugettext as _, ugettext as err_ +from django.utils.translation import ugettext as _ +from django.utils.translation import ugettext as err_ from django.views.decorators.csrf import csrf_exempt from analytics.lib.counts import COUNT_STATS -from zerver.decorator import require_post, InvalidZulipServerKeyError +from zerver.decorator import InvalidZulipServerKeyError, require_post from zerver.lib.exceptions import JsonableError -from zerver.lib.push_notifications import send_android_push_notification, \ - send_apple_push_notification +from zerver.lib.push_notifications import ( + send_android_push_notification, + send_apple_push_notification, +) from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_error, json_success -from zerver.lib.validator import check_int, check_string, \ - check_capped_string, check_string_fixed_length, check_float, check_none_or, \ - check_dict_only, check_list, check_bool +from zerver.lib.validator import ( + check_bool, + check_capped_string, + check_dict_only, + check_float, + check_int, + check_list, + check_none_or, + check_string, + check_string_fixed_length, +) from zerver.models import UserProfile from zerver.views.push_notifications import validate_token -from zilencer.models import RemotePushDeviceToken, RemoteZulipServer, \ - RemoteRealmCount, RemoteInstallationCount, RemoteRealmAuditLog +from zilencer.models import ( + RemoteInstallationCount, + RemotePushDeviceToken, + RemoteRealmAuditLog, + RemoteRealmCount, + RemoteZulipServer, +) + def validate_entity(entity: Union[UserProfile, RemoteZulipServer]) -> RemoteZulipServer: if not isinstance(entity, RemoteZulipServer): diff --git a/zproject/backends.py b/zproject/backends.py index 56e6b11fe4375..b53aac21b9e77 100644 --- a/zproject/backends.py +++ b/zproject/backends.py @@ -15,67 +15,81 @@ import binascii import copy import logging +from abc import ABC, abstractmethod +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, TypeVar, Union, cast + import jwt import magic import ujson -from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, TypeVar, Union, \ - cast -from typing_extensions import TypedDict -from zxcvbn import zxcvbn - -from django_auth_ldap.backend import LDAPBackend, LDAPReverseEmailSearch, \ - _LDAPUser, ldap_error from decorator import decorator - +from django.conf import settings from django.contrib.auth import authenticate, get_backends from django.contrib.auth.backends import RemoteUserBackend -from django.conf import settings from django.core.exceptions import ValidationError from django.core.validators import validate_email -from django.dispatch import receiver, Signal -from django.http import HttpResponse, HttpResponseRedirect, HttpRequest +from django.dispatch import Signal, receiver +from django.http import HttpRequest, HttpResponse, HttpResponseRedirect from django.shortcuts import render from django.urls import reverse from django.utils.translation import ugettext as _ +from django_auth_ldap.backend import LDAPBackend, LDAPReverseEmailSearch, _LDAPUser, ldap_error from jwt.algorithms import RSAAlgorithm from jwt.exceptions import PyJWTError from lxml.etree import XMLSyntaxError -from requests import HTTPError from onelogin.saml2.errors import OneLogin_Saml2_Error from onelogin.saml2.response import OneLogin_Saml2_Response - -from social_core.backends.github import GithubOAuth2, GithubOrganizationOAuth2, \ - GithubTeamOAuth2 +from requests import HTTPError +from social_core.backends.apple import AppleIdAuth from social_core.backends.azuread import AzureADOAuth2 -from social_core.backends.gitlab import GitLabOAuth2 from social_core.backends.base import BaseAuth +from social_core.backends.github import GithubOAuth2, GithubOrganizationOAuth2, GithubTeamOAuth2 +from social_core.backends.gitlab import GitLabOAuth2 from social_core.backends.google import GoogleOAuth2 -from social_core.backends.apple import AppleIdAuth from social_core.backends.saml import SAMLAuth +from social_core.exceptions import ( + AuthFailed, + AuthMissingParameter, + AuthStateForbidden, + SocialAuthBaseException, +) from social_core.pipeline.partial import partial -from social_core.exceptions import AuthFailed, SocialAuthBaseException, \ - AuthMissingParameter, AuthStateForbidden +from typing_extensions import TypedDict +from zxcvbn import zxcvbn from zerver.decorator import client_is_exempt_from_rate_limiting -from zerver.lib.actions import do_create_user, do_reactivate_user, do_deactivate_user, \ - do_update_user_custom_profile_data_if_changed -from zerver.lib.avatar import is_avatar_new, avatar_url +from zerver.lib.actions import ( + do_create_user, + do_deactivate_user, + do_reactivate_user, + do_update_user_custom_profile_data_if_changed, +) +from zerver.lib.avatar import avatar_url, is_avatar_new from zerver.lib.avatar_hash import user_avatar_content_hash from zerver.lib.create_user import get_role_for_new_user from zerver.lib.dev_ldap_directory import init_fakeldap -from zerver.lib.email_validation import email_allowed_for_realm, \ - validate_email_not_already_in_realm +from zerver.lib.email_validation import email_allowed_for_realm, validate_email_not_already_in_realm from zerver.lib.mobile_auth_otp import is_valid_otp from zerver.lib.rate_limiter import RateLimitedObject +from zerver.lib.redis_utils import get_dict_from_redis, get_redis_client, put_dict_in_redis from zerver.lib.request import JsonableError -from zerver.lib.users import check_full_name, validate_user_custom_profile_field -from zerver.lib.redis_utils import get_redis_client, get_dict_from_redis, put_dict_in_redis from zerver.lib.subdomains import get_subdomain -from zerver.models import CustomProfileField, DisposableEmailError, DomainNotAllowedForRealmError, \ - EmailContainsPlusError, PreregistrationUser, UserProfile, Realm, custom_profile_fields_for_realm, \ - get_user_profile_by_id, remote_user_to_email, \ - email_to_username, get_realm, get_user_by_delivery_email, supported_auth_backends +from zerver.lib.users import check_full_name, validate_user_custom_profile_field +from zerver.models import ( + CustomProfileField, + DisposableEmailError, + DomainNotAllowedForRealmError, + EmailContainsPlusError, + PreregistrationUser, + Realm, + UserProfile, + custom_profile_fields_for_realm, + email_to_username, + get_realm, + get_user_by_delivery_email, + get_user_profile_by_id, + remote_user_to_email, + supported_auth_backends, +) redis_client = get_redis_client() @@ -505,10 +519,11 @@ def ldap_to_django_username(self, username: str) -> str: def sync_avatar_from_ldap(self, user: UserProfile, ldap_user: _LDAPUser) -> None: if 'avatar' in settings.AUTH_LDAP_USER_ATTR_MAP: # We do local imports here to avoid import loops - from zerver.lib.upload import upload_avatar_image - from zerver.lib.actions import do_change_avatar_fields from io import BytesIO + from zerver.lib.actions import do_change_avatar_fields + from zerver.lib.upload import upload_avatar_image + avatar_attr_name = settings.AUTH_LDAP_USER_ATTR_MAP['avatar'] if avatar_attr_name not in ldap_user.attrs: # nocoverage # If this specific user doesn't have e.g. a @@ -1236,8 +1251,7 @@ def social_auth_finish(backend: Any, comments below as well as login_or_register_remote_user in `zerver/views/auth.py` for the details on how that dispatch works. """ - from zerver.views.auth import (login_or_register_remote_user, - redirect_and_log_into_subdomain) + from zerver.views.auth import login_or_register_remote_user, redirect_and_log_into_subdomain user_profile = kwargs['user_profile'] return_data = kwargs['return_data'] diff --git a/zproject/computed_settings.py b/zproject/computed_settings.py index 7b2be1e10b095..f456f6293a137 100644 --- a/zproject/computed_settings.py +++ b/zproject/computed_settings.py @@ -1,21 +1,28 @@ -from copy import deepcopy import os -import time import sys +import time +from copy import deepcopy from typing import Any, Dict, List, Optional, Tuple, Union from urllib.parse import urljoin -from zerver.lib.db import TimeTrackingConnection import zerver.lib.logging_util +from zerver.lib.db import TimeTrackingConnection -from .config import DEPLOY_ROOT, PRODUCTION, DEVELOPMENT, get_secret, get_config, get_from_file_if_exists +from .config import ( + DEPLOY_ROOT, + DEVELOPMENT, + PRODUCTION, + get_config, + get_from_file_if_exists, + get_secret, +) from .configured_settings import ( ADMINS, ALLOWED_HOSTS, - AUTHENTICATION_BACKENDS, AUTH_LDAP_BIND_DN, AUTH_LDAP_CONNECTION_OPTIONS, AUTH_LDAP_SERVER_URI, + AUTHENTICATION_BACKENDS, CAMO_URI, DEBUG, DEBUG_ERROR_REPORTING, @@ -156,6 +163,8 @@ ALLOWED_HOSTS += REALM_HOSTS.values() from django.template.loaders import app_directories + + class TwoFactorLoader(app_directories.Loader): def get_dirs(self) -> List[str]: dirs = super().get_dirs() diff --git a/zproject/config.py b/zproject/config.py index f55c616c1c193..1a10b7f1f8d47 100644 --- a/zproject/config.py +++ b/zproject/config.py @@ -1,6 +1,6 @@ +import configparser import os from typing import Optional, overload -import configparser DEPLOY_ROOT = os.path.realpath(os.path.dirname(os.path.dirname(__file__))) diff --git a/zproject/configured_settings.py b/zproject/configured_settings.py index 10997c8193d9c..fc4f128e834e2 100644 --- a/zproject/configured_settings.py +++ b/zproject/configured_settings.py @@ -10,6 +10,7 @@ # Import variables like secrets from the prod_settings file # Import prod_settings after determining the deployment/machine type from .config import PRODUCTION + if PRODUCTION: from .prod_settings import * # noqa: F401,F403 isort: skip else: diff --git a/zproject/default_settings.py b/zproject/default_settings.py index 84285801a9969..e9cccd61d6efb 100644 --- a/zproject/default_settings.py +++ b/zproject/default_settings.py @@ -1,10 +1,11 @@ -from typing import Any, Dict, List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict, List, Optional if TYPE_CHECKING: from django_auth_ldap.config import LDAPSearch from typing_extensions import TypedDict -from .config import PRODUCTION, DEVELOPMENT, get_secret +from .config import DEVELOPMENT, PRODUCTION, get_secret + if PRODUCTION: from .prod_settings import EXTERNAL_HOST, ZULIP_ADMINISTRATOR else: diff --git a/zproject/dev_settings.py b/zproject/dev_settings.py index 3cd628a886854..c551372800b6a 100644 --- a/zproject/dev_settings.py +++ b/zproject/dev_settings.py @@ -113,6 +113,7 @@ if FAKE_LDAP_MODE: import ldap from django_auth_ldap.config import LDAPSearch + # To understand these parameters, read the docs in # prod_settings_template.py and on ReadTheDocs. LDAP_APPEND_DOMAIN = None diff --git a/zproject/dev_urls.py b/zproject/dev_urls.py index 254b17c981595..2b0ea06b62ebc 100644 --- a/zproject/dev_urls.py +++ b/zproject/dev_urls.py @@ -1,16 +1,18 @@ +import os from urllib.parse import urlsplit -from django.conf.urls import url + from django.conf import settings +from django.conf.urls import url from django.conf.urls.static import static from django.contrib.staticfiles.views import serve as staticfiles_serve from django.http import HttpRequest, HttpResponse from django.views.generic import TemplateView -import os from django.views.static import serve -import zerver.views.development.registration + import zerver.views.auth import zerver.views.development.email_log import zerver.views.development.integrations +import zerver.views.development.registration # These URLs are available only in the development environment diff --git a/zproject/email_backends.py b/zproject/email_backends.py index 021569435367a..9a082088cc4dd 100644 --- a/zproject/email_backends.py +++ b/zproject/email_backends.py @@ -1,17 +1,16 @@ -import logging - -from typing import List import configparser - +import logging import smtplib from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText +from typing import List from django.conf import settings -from django.core.mail.backends.base import BaseEmailBackend from django.core.mail import EmailMultiAlternatives +from django.core.mail.backends.base import BaseEmailBackend from django.template import loader + def get_forward_address() -> str: config = configparser.ConfigParser() config.read(settings.FORWARD_ADDRESS_CONFIG_FILE) diff --git a/zproject/jinja2/__init__.py b/zproject/jinja2/__init__.py index 138a4ae683150..fb8582a2fe271 100644 --- a/zproject/jinja2/__init__.py +++ b/zproject/jinja2/__init__.py @@ -2,7 +2,7 @@ from django.conf import settings from django.contrib.staticfiles.storage import staticfiles_storage -from django.template.defaultfilters import slugify, pluralize +from django.template.defaultfilters import pluralize, slugify from django.urls import reverse from django.utils import translation from django.utils.timesince import timesince diff --git a/zproject/legacy_urls.py b/zproject/legacy_urls.py index b7f7a70f3688d..1366f13be0d60 100644 --- a/zproject/legacy_urls.py +++ b/zproject/legacy_urls.py @@ -1,9 +1,10 @@ from django.urls import path + import zerver.views -import zerver.views.streams import zerver.views.auth -import zerver.views.tutorial import zerver.views.report +import zerver.views.streams +import zerver.views.tutorial # Future endpoints should add to urls.py, which includes these legacy urls diff --git a/zproject/test_extra_settings.py b/zproject/test_extra_settings.py index 6a6bfd2753915..d7d414f275c82 100644 --- a/zproject/test_extra_settings.py +++ b/zproject/test_extra_settings.py @@ -5,6 +5,7 @@ from django_auth_ldap.config import LDAPSearch from zerver.lib.db import TimeTrackingConnection + from .config import DEPLOY_ROOT, get_from_file_if_exists from .settings import ( AUTHENTICATION_BACKENDS, diff --git a/zproject/test_settings.py b/zproject/test_settings.py index 28522199e6395..0f10b85840bd2 100644 --- a/zproject/test_settings.py +++ b/zproject/test_settings.py @@ -1,4 +1,5 @@ import os + # test_settings.py works differently from # dev_settings.py/prod_settings.py; it actually is directly referenced # by the test suite as DJANGO_SETTINGS_MODULE and imports settings.py diff --git a/zproject/urls.py b/zproject/urls.py index a3f6bc1ba7e75..8349b02c26c26 100644 --- a/zproject/urls.py +++ b/zproject/urls.py @@ -1,49 +1,51 @@ +import os + from django.conf import settings -from django.conf.urls import url, include +from django.conf.urls import include, url from django.conf.urls.i18n import i18n_patterns -from django.views.generic import TemplateView, RedirectView +from django.contrib.auth.views import ( + LoginView, + PasswordResetCompleteView, + PasswordResetConfirmView, + PasswordResetDoneView, +) from django.utils.module_loading import import_string -import os -import zerver.forms -from zproject import dev_urls -from zproject.legacy_urls import legacy_urls -from zerver.views.documentation import IntegrationView, MarkdownDirectoryView -from zerver.lib.integrations import WEBHOOK_INTEGRATIONS - - -from django.contrib.auth.views import (LoginView, PasswordResetDoneView, - PasswordResetConfirmView, PasswordResetCompleteView) +from django.views.generic import RedirectView, TemplateView +import zerver.forms import zerver.tornado.views import zerver.views -import zerver.views.auth import zerver.views.archive +import zerver.views.auth import zerver.views.camo import zerver.views.compatibility -import zerver.views.home -import zerver.views.email_mirror -import zerver.views.registration -import zerver.views.portico -import zerver.views.zephyr -import zerver.views.users -import zerver.views.unsubscribe +import zerver.views.digest import zerver.views.documentation -import zerver.views.user_groups -import zerver.views.user_settings +import zerver.views.email_mirror +import zerver.views.home +import zerver.views.messages import zerver.views.muting -import zerver.views.streams +import zerver.views.portico import zerver.views.realm -import zerver.views.digest -import zerver.views.messages import zerver.views.realm_export +import zerver.views.registration +import zerver.views.streams +import zerver.views.unsubscribe import zerver.views.upload +import zerver.views.user_groups +import zerver.views.user_settings +import zerver.views.users import zerver.views.video_calls - +import zerver.views.zephyr +from zerver.lib.integrations import WEBHOOK_INTEGRATIONS from zerver.lib.rest import rest_dispatch +from zerver.views.documentation import IntegrationView, MarkdownDirectoryView +from zproject import dev_urls +from zproject.legacy_urls import legacy_urls if settings.TWO_FACTOR_AUTHENTICATION_ENABLED: - from two_factor.urls import urlpatterns as tf_urls from two_factor.gateways.twilio.urls import urlpatterns as tf_twilio_urls + from two_factor.urls import urlpatterns as tf_urls # NB: There are several other pieces of code which route requests by URL: # diff --git a/zproject/wsgi.py b/zproject/wsgi.py index 0d1253c779150..75144d5732574 100644 --- a/zproject/wsgi.py +++ b/zproject/wsgi.py @@ -25,6 +25,7 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings") import django + try: django.setup() except Exception as e: @@ -45,10 +46,12 @@ # need to import zerver.models first before the middleware tries to import it. import zerver.models + zerver.models # This application object is used by any WSGI server configured to use this # file. This includes Django's development server, if the WSGI_APPLICATION # setting points here. from django.core.wsgi import get_wsgi_application + application = get_wsgi_application() diff --git a/zthumbor/loaders/helpers.py b/zthumbor/loaders/helpers.py index 615e4d77c8a04..b595256dec012 100644 --- a/zthumbor/loaders/helpers.py +++ b/zthumbor/loaders/helpers.py @@ -4,7 +4,7 @@ import os import re import sys -from typing import Any, Text, Tuple, Optional +from typing import Any, Optional, Text, Tuple ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) diff --git a/zthumbor/loaders/zloader.py b/zthumbor/loaders/zloader.py index 98bd506d777d5..74f694273a43e 100644 --- a/zthumbor/loaders/zloader.py +++ b/zthumbor/loaders/zloader.py @@ -1,20 +1,24 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/thumbnailing.html from __future__ import absolute_import +import base64 +import logging +from typing import Any, Callable + from six.moves import urllib from tornado.concurrent import return_future -from thumbor.loaders import LoaderResult, file_loader, https_loader + from tc_aws.loaders import s3_loader from thumbor.context import Context +from thumbor.loaders import LoaderResult, file_loader, https_loader + from .helpers import ( + THUMBOR_EXTERNAL_TYPE, + THUMBOR_LOCAL_FILE_TYPE, + THUMBOR_S3_TYPE, separate_url_and_source_type, - THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE, THUMBOR_EXTERNAL_TYPE, ) -from typing import Any, Callable - -import base64 -import logging def get_not_found_result(): # type: () -> LoaderResult