diff --git a/src/config.ts b/src/config.ts
index dc7512520..db09b7da6 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -3,7 +3,7 @@ import { Context } from 'probot'
 import getConfig from 'probot-config'
 import { Decoder, object, string, optional, number, boolean, array, oneOf, constant } from '@mojotech/json-type-validation'
 
-class ConfigNotFoundError extends Error {
+export class ConfigNotFoundError extends Error {
   constructor (
     public readonly filePath: string
   ) {
diff --git a/src/index.ts b/src/index.ts
index 9c0fa0a8b..83dd55928 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -1,5 +1,5 @@
 import { Application, Context } from 'probot'
-import { loadConfig } from './config'
+import { loadConfig, ConfigNotFoundError, ConfigValidationError } from './config'
 import { WorkerContext } from './models'
 import Raven from 'raven'
 import { RepositoryWorkers } from './repository-workers'
@@ -33,7 +33,19 @@ async function useWorkerContext (options: {app: Application, context: Context, i
       event: options.context.event
     }
   }, async () => {
-    const workerContext = await getWorkerContext(options)
+    let workerContext
+    try {
+      workerContext = await getWorkerContext(options)
+    } catch (err) {
+      if (err instanceof ConfigNotFoundError || err instanceof ConfigValidationError) {
+        // Skip worker callback as we cannot create a context. We also do not raise this error as this is part of
+        // normal behaviour.
+        return
+      } else {
+        throw err
+      }
+    }
+
     await fn(workerContext)
   })
 }
diff --git a/test/integration.test.ts b/test/integration.test.ts
index 4fa64d3ec..63525d1e6 100644
--- a/test/integration.test.ts
+++ b/test/integration.test.ts
@@ -126,9 +126,10 @@ it('not enough approval reviews', async () => {
 })
 
 it('no configuration should not schedule any pull request', async () => {
+  const schedulePullRequestTrigger = jest.fn()
   jest.mock('../src/pull-request-handler', () => {
     return {
-      schedulePullRequestTrigger: jest.fn()
+      schedulePullRequestTrigger
     }
   })
 
@@ -144,15 +145,15 @@ it('no configuration should not schedule any pull request', async () => {
     github
   })
 
-  expect(
-    app.receive(
-      createPullRequestOpenedEvent({
-        owner: 'bobvanderlinden',
-        repo: 'probot-auto-merge',
-        number: 1
-      })
-    )
-  ).rejects.toHaveProperty('message', "Configuration file '.github/auto-merge.yml' not found")
+  app.receive(
+    createPullRequestOpenedEvent({
+      owner: 'bobvanderlinden',
+      repo: 'probot-auto-merge',
+      number: 1
+    })
+  )
+
+  expect(schedulePullRequestTrigger).toBeCalledTimes(0)
 })
 
 it('merges when receiving status event', async () => {