Sneakpeek
0.1
Overview
Quick start
Local handler debugging
Design
Deployment options
Plugins
Rate limiter
Robots.txt
User Agent injector
Proxy plugin
Requests logging middleware
Adding your own plugin
API
Sneakpeek
»
Index
Index
A
|
B
|
C
|
D
|
E
|
F
|
G
|
H
|
I
|
K
|
L
|
M
|
N
|
O
|
P
|
Q
|
R
|
S
|
T
|
U
|
W
A
acquired (sneakpeek.models.Lease attribute)
acquired_until (sneakpeek.models.Lease attribute)
add_scraper_job() (sneakpeek.storage.base.ScraperJobsStorage method)
(sneakpeek.storage.in_memory_storage.InMemoryScraperJobsStorage method)
(sneakpeek.storage.redis_storage.RedisScraperJobsStorage method)
after_response() (sneakpeek.plugins.requests_logging_plugin.RequestsLoggingPlugin method)
(sneakpeek.scraper_context.AfterResponsePlugin method)
AfterResponsePlugin (class in sneakpeek.scraper_context)
B
before_request() (sneakpeek.plugins.proxy_plugin.ProxyPlugin method)
(sneakpeek.plugins.rate_limiter_plugin.RateLimiterPlugin method)
(sneakpeek.plugins.requests_logging_plugin.RequestsLoggingPlugin method)
(sneakpeek.plugins.robots_txt_plugin.RobotsTxtPlugin method)
(sneakpeek.plugins.user_agent_injecter_plugin.UserAgentInjecterPlugin method)
(sneakpeek.scraper_context.BeforeRequestPlugin method)
BeforeRequestPlugin (class in sneakpeek.scraper_context)
browsers (sneakpeek.plugins.user_agent_injecter_plugin.UserAgentInjecterPluginConfig attribute)
C
config (sneakpeek.models.Scraper attribute)
configure_logging() (in module sneakpeek.logging)
count_invocations() (in module sneakpeek.metrics)
create() (sneakpeek.server.SneakpeekServer static method)
create_api() (in module sneakpeek.api)
create_scraper() (sneakpeek.storage.base.ScrapersStorage method)
(sneakpeek.storage.in_memory_storage.InMemoryScrapersStorage method)
(sneakpeek.storage.redis_storage.RedisScrapersStorage method)
created_at (sneakpeek.models.ScraperJob attribute)
CRONTAB (sneakpeek.models.ScraperSchedule attribute)
D
DEAD (sneakpeek.models.ScraperJobStatus attribute)
delete() (sneakpeek.scraper_context.ScraperContext method)
delete_old_scraper_jobs() (sneakpeek.storage.base.ScraperJobsStorage method)
(sneakpeek.storage.in_memory_storage.InMemoryScraperJobsStorage method)
(sneakpeek.storage.redis_storage.RedisScraperJobsStorage method)
delete_scraper() (sneakpeek.storage.base.ScrapersStorage method)
(sneakpeek.storage.in_memory_storage.InMemoryScrapersStorage method)
(sneakpeek.storage.redis_storage.RedisScrapersStorage method)
dequeue() (sneakpeek.queue.QueueABC method)
dequeue_scraper_job() (sneakpeek.storage.base.ScraperJobsStorage method)
(sneakpeek.storage.in_memory_storage.InMemoryScraperJobsStorage method)
(sneakpeek.storage.redis_storage.RedisScraperJobsStorage method)
download_file() (sneakpeek.scraper_context.ScraperContext method)
download_files() (sneakpeek.scraper_context.ScraperContext method)
E
enqueue() (sneakpeek.queue.QueueABC method)
EVERY_DAY (sneakpeek.models.ScraperSchedule attribute)
EVERY_HOUR (sneakpeek.models.ScraperSchedule attribute)
EVERY_MINUTE (sneakpeek.models.ScraperSchedule attribute)
EVERY_MONTH (sneakpeek.models.ScraperSchedule attribute)
EVERY_SECOND (sneakpeek.models.ScraperSchedule attribute)
EVERY_WEEK (sneakpeek.models.ScraperSchedule attribute)
F
FAILED (sneakpeek.models.ScraperJobStatus attribute)
filter() (sneakpeek.logging.ScraperContextInjectingFilter method)
finished_at (sneakpeek.models.ScraperJob attribute)
full_match (sneakpeek.scraper_context.RegexMatch attribute)
G
get() (sneakpeek.scraper_context.ScraperContext method)
get_api_entrypoint() (in module sneakpeek.api)
get_queue_len() (sneakpeek.queue.QueueABC method)
(sneakpeek.storage.base.ScraperJobsStorage method)
(sneakpeek.storage.in_memory_storage.InMemoryScraperJobsStorage method)
(sneakpeek.storage.redis_storage.RedisScraperJobsStorage method)
get_scraper() (sneakpeek.storage.base.ScrapersStorage method)
(sneakpeek.storage.in_memory_storage.InMemoryScrapersStorage method)
(sneakpeek.storage.redis_storage.RedisScrapersStorage method)
get_scraper_job() (sneakpeek.storage.base.ScraperJobsStorage method)
(sneakpeek.storage.in_memory_storage.InMemoryScraperJobsStorage method)
(sneakpeek.storage.redis_storage.RedisScraperJobsStorage method)
get_scraper_jobs() (sneakpeek.storage.base.ScraperJobsStorage method)
(sneakpeek.storage.in_memory_storage.InMemoryScraperJobsStorage method)
(sneakpeek.storage.redis_storage.RedisScraperJobsStorage method)
get_scrapers() (sneakpeek.storage.base.ScrapersStorage method)
(sneakpeek.storage.in_memory_storage.InMemoryScrapersStorage method)
(sneakpeek.storage.redis_storage.RedisScrapersStorage method)
groups (sneakpeek.scraper_context.RegexMatch attribute)
H
handler (sneakpeek.models.Scraper attribute)
head() (sneakpeek.scraper_context.ScraperContext method)
HIGH (sneakpeek.models.ScraperJobPriority attribute)
HttpMethod (class in sneakpeek.scraper_context)
I
id (sneakpeek.models.Scraper attribute)
(sneakpeek.models.ScraperJob attribute)
INACTIVE (sneakpeek.models.ScraperSchedule attribute)
InMemoryLeaseStorage (class in sneakpeek.storage.in_memory_storage)
InMemoryScraperJobsStorage (class in sneakpeek.storage.in_memory_storage)
InMemoryScrapersStorage (class in sneakpeek.storage.in_memory_storage)
is_read_only() (sneakpeek.storage.base.ScrapersStorage method)
(sneakpeek.storage.in_memory_storage.InMemoryScrapersStorage method)
(sneakpeek.storage.redis_storage.RedisScrapersStorage method)
K
kill_dead_scraper_jobs() (sneakpeek.queue.QueueABC method)
KILLED (sneakpeek.models.ScraperJobStatus attribute)
L
last_active_at (sneakpeek.models.ScraperJob attribute)
Lease (class in sneakpeek.models)
LeaseStorage (class in sneakpeek.storage.base)
LocalRunner (class in sneakpeek.runner)
LOG (sneakpeek.plugins.robots_txt_plugin.RobotsTxtViolationStrategy attribute)
log_request (sneakpeek.plugins.requests_logging_plugin.RequestsLoggingPluginConfig attribute)
log_response (sneakpeek.plugins.requests_logging_plugin.RequestsLoggingPluginConfig attribute)
M
max_requests (sneakpeek.plugins.rate_limiter_plugin.RateLimiterPluginConfig attribute)
maybe_acquire_lease() (sneakpeek.storage.base.LeaseStorage method)
(sneakpeek.storage.in_memory_storage.InMemoryLeaseStorage method)
(sneakpeek.storage.redis_storage.RedisLeaseStorage method)
maybe_get_scraper() (sneakpeek.storage.base.ScrapersStorage method)
(sneakpeek.storage.in_memory_storage.InMemoryScrapersStorage method)
(sneakpeek.storage.redis_storage.RedisScrapersStorage method)
measure_latency() (in module sneakpeek.metrics)
module
sneakpeek.api
sneakpeek.errors
sneakpeek.logging
sneakpeek.metrics
sneakpeek.models
sneakpeek.plugins.proxy_plugin
sneakpeek.plugins.rate_limiter_plugin
sneakpeek.plugins.requests_logging_plugin
sneakpeek.plugins.robots_txt_plugin
sneakpeek.plugins.user_agent_injecter_plugin
sneakpeek.plugins.utils
sneakpeek.queue
sneakpeek.runner
sneakpeek.scheduler
sneakpeek.scraper_config
sneakpeek.scraper_context
sneakpeek.scraper_handler
sneakpeek.server
sneakpeek.storage.base
sneakpeek.storage.in_memory_storage
sneakpeek.storage.redis_storage
sneakpeek.worker
N
name (sneakpeek.models.Lease attribute)
(sneakpeek.models.Scraper attribute)
(sneakpeek.plugins.proxy_plugin.ProxyPlugin property)
(sneakpeek.plugins.rate_limiter_plugin.RateLimiterPlugin property)
(sneakpeek.plugins.requests_logging_plugin.RequestsLoggingPlugin property)
(sneakpeek.plugins.robots_txt_plugin.RobotsTxtPlugin property)
(sneakpeek.plugins.user_agent_injecter_plugin.UserAgentInjecterPlugin property)
(sneakpeek.scraper_context.AfterResponsePlugin property)
(sneakpeek.scraper_context.BeforeRequestPlugin property)
(sneakpeek.scraper_handler.ScraperHandler property)
NORMAL (sneakpeek.models.ScraperJobPriority attribute)
O
options() (sneakpeek.scraper_context.ScraperContext method)
owner_id (sneakpeek.models.Lease attribute)
P
params (sneakpeek.scraper_config.ScraperConfig attribute)
PENDING (sneakpeek.models.ScraperJobStatus attribute)
ping_scraper_job() (sneakpeek.queue.QueueABC method)
plugins (sneakpeek.scraper_config.ScraperConfig attribute)
post() (sneakpeek.scraper_context.ScraperContext method)
priority (sneakpeek.models.ScraperJob attribute)
proxy (sneakpeek.plugins.proxy_plugin.ProxyPluginConfig attribute)
proxy_auth (sneakpeek.plugins.proxy_plugin.ProxyPluginConfig attribute)
ProxyPlugin (class in sneakpeek.plugins.proxy_plugin)
ProxyPluginConfig (class in sneakpeek.plugins.proxy_plugin)
put() (sneakpeek.scraper_context.ScraperContext method)
Q
Queue (class in sneakpeek.queue)
QueueABC (class in sneakpeek.queue)
R
rate_limited_strategy (sneakpeek.plugins.rate_limiter_plugin.RateLimiterPluginConfig attribute)
RateLimitedException
RateLimitedStrategy (class in sneakpeek.plugins.rate_limiter_plugin)
RateLimiterPlugin (class in sneakpeek.plugins.rate_limiter_plugin)
RateLimiterPluginConfig (class in sneakpeek.plugins.rate_limiter_plugin)
RedisLeaseStorage (class in sneakpeek.storage.redis_storage)
RedisScraperJobsStorage (class in sneakpeek.storage.redis_storage)
RedisScrapersStorage (class in sneakpeek.storage.redis_storage)
regex() (sneakpeek.scraper_context.ScraperContext method)
RegexMatch (class in sneakpeek.scraper_context)
release_lease() (sneakpeek.storage.base.LeaseStorage method)
(sneakpeek.storage.in_memory_storage.InMemoryLeaseStorage method)
(sneakpeek.storage.redis_storage.RedisLeaseStorage method)
Request (class in sneakpeek.scraper_context)
request() (sneakpeek.scraper_context.ScraperContext method)
RequestsLoggingPlugin (class in sneakpeek.plugins.requests_logging_plugin)
RequestsLoggingPluginConfig (class in sneakpeek.plugins.requests_logging_plugin)
result (sneakpeek.models.ScraperJob attribute)
RobotsTxtPlugin (class in sneakpeek.plugins.robots_txt_plugin)
RobotsTxtPluginConfig (class in sneakpeek.plugins.robots_txt_plugin)
RobotsTxtViolationException
RobotsTxtViolationStrategy (class in sneakpeek.plugins.robots_txt_plugin)
run() (sneakpeek.runner.LocalRunner static method)
(sneakpeek.runner.Runner method)
(sneakpeek.runner.RunnerABC method)
(sneakpeek.scraper_handler.ScraperHandler method)
run_async() (sneakpeek.runner.LocalRunner static method)
Runner (class in sneakpeek.runner)
RunnerABC (class in sneakpeek.runner)
S
schedule (sneakpeek.models.Scraper attribute)
schedule_crontab (sneakpeek.models.Scraper attribute)
schedule_priority (sneakpeek.models.Scraper attribute)
Scheduler (class in sneakpeek.scheduler)
SchedulerABC (class in sneakpeek.scheduler)
Scraper (class in sneakpeek.models)
scraper (sneakpeek.models.ScraperJob attribute)
scraper_job_context() (in module sneakpeek.logging)
ScraperConfig (class in sneakpeek.scraper_config)
ScraperContext (class in sneakpeek.scraper_context)
ScraperContextInjectingFilter (class in sneakpeek.logging)
ScraperHandler (class in sneakpeek.scraper_handler)
ScraperHasActiveRunError
ScraperJob (class in sneakpeek.models)
ScraperJobNotFoundError
ScraperJobPingFinishedError
ScraperJobPingNotStartedError
ScraperJobPriority (class in sneakpeek.models)
ScraperJobsStorage (class in sneakpeek.storage.base)
ScraperJobStatus (class in sneakpeek.models)
ScraperJobTimedOut
ScraperNotFoundError
ScraperSchedule (class in sneakpeek.models)
ScrapersStorage (class in sneakpeek.storage.base)
search_scrapers() (sneakpeek.storage.base.ScrapersStorage method)
(sneakpeek.storage.in_memory_storage.InMemoryScrapersStorage method)
(sneakpeek.storage.redis_storage.RedisScrapersStorage method)
serve() (sneakpeek.server.SneakpeekServer method)
sneakpeek.api
module
sneakpeek.errors
module
sneakpeek.logging
module
sneakpeek.metrics
module
sneakpeek.models
module
sneakpeek.plugins.proxy_plugin
module
sneakpeek.plugins.rate_limiter_plugin
module
sneakpeek.plugins.requests_logging_plugin
module
sneakpeek.plugins.robots_txt_plugin
module
sneakpeek.plugins.user_agent_injecter_plugin
module
sneakpeek.plugins.utils
module
sneakpeek.queue
module
sneakpeek.runner
module
sneakpeek.scheduler
module
sneakpeek.scraper_config
module
sneakpeek.scraper_context
module
sneakpeek.scraper_handler
module
sneakpeek.server
module
sneakpeek.storage.base
module
sneakpeek.storage.in_memory_storage
module
sneakpeek.storage.redis_storage
module
sneakpeek.worker
module
SneakpeekServer (class in sneakpeek.server)
STARTED (sneakpeek.models.ScraperJobStatus attribute)
started_at (sneakpeek.models.ScraperJob attribute)
state (sneakpeek.models.Scraper attribute)
status (sneakpeek.models.ScraperJob attribute)
stop() (sneakpeek.server.SneakpeekServer method)
SUCCEEDED (sneakpeek.models.ScraperJobStatus attribute)
T
THROW (sneakpeek.plugins.rate_limiter_plugin.RateLimitedStrategy attribute)
(sneakpeek.plugins.robots_txt_plugin.RobotsTxtViolationStrategy attribute)
time_window (sneakpeek.plugins.rate_limiter_plugin.RateLimiterPluginConfig attribute)
timeout_seconds (sneakpeek.models.Scraper attribute)
U
UnknownScraperHandlerError
update_scraper() (sneakpeek.storage.base.ScrapersStorage method)
(sneakpeek.storage.in_memory_storage.InMemoryScrapersStorage method)
(sneakpeek.storage.redis_storage.RedisScrapersStorage method)
update_scraper_job() (sneakpeek.storage.base.ScraperJobsStorage method)
(sneakpeek.storage.in_memory_storage.InMemoryScraperJobsStorage method)
(sneakpeek.storage.redis_storage.RedisScraperJobsStorage method)
update_scraper_state() (sneakpeek.scraper_context.ScraperContext method)
use_external_data (sneakpeek.plugins.user_agent_injecter_plugin.UserAgentInjecterPluginConfig attribute)
UserAgentInjecterPlugin (class in sneakpeek.plugins.user_agent_injecter_plugin)
UserAgentInjecterPluginConfig (class in sneakpeek.plugins.user_agent_injecter_plugin)
UTMOST (sneakpeek.models.ScraperJobPriority attribute)
W
WAIT (sneakpeek.plugins.rate_limiter_plugin.RateLimitedStrategy attribute)
Worker (class in sneakpeek.worker)
WorkerABC (class in sneakpeek.worker)