I've used a script to run selenium locally so that I can make use of the response (derived from selenium) within my spider.

This is the web service where selenium runs locally:

from flask import Flask, request, make_response
from flask_restful import Resource, Api
from selenium import webdriver
from selenium.webdriver.chrome.options import Options

app = Flask(__name__)
api = Api(app)

class Selenium(Resource):
    _driver = None

    @staticmethod
    def getDriver():
        if not Selenium._driver:
            chrome_options = Options()
            chrome_options.add_argument("--headless")

            Selenium._driver = webdriver.Chrome(options=chrome_options)
        return Selenium._driver

    @property
    def driver(self):
        return Selenium.getDriver()

    def get(self):
        url = str(request.args['url'])

        self.driver.get(url)

        return make_response(self.driver.page_source)

api.add_resource(Selenium, '/')

if __name__ == '__main__':
    app.run(debug=True)

This is my scrapy spider which takes the benefit of that response to parse the title from a webpage.

import scrapy
from urllib.parse import quote
from scrapy.crawler import CrawlerProcess

class StackSpider(scrapy.Spider):
    name = 'stackoverflow'
    url = 'https://stackoverflow.com/questions/tagged/web-scraping?sort=newest&pageSize=50'
    base = 'https://stackoverflow.com'

    def start_requests(self):
        link = 'http://127.0.0.1:5000/?url={}'.format(quote(self.url))
        yield scrapy.Request(link,callback=self.parse)

    def parse(self, response):
        for item in response.css(".summary .question-hyperlink::attr(href)").getall():
            nlink = self.base + item
            link = 'http://127.0.0.1:5000/?url={}'.format(quote(nlink))
            yield scrapy.Request(link,callback=self.parse_info,dont_filter=True)

    def parse_info(self, response):
        item = response.css('h1[itemprop="name"] > a::text').get()
        yield {"title":item}

if __name__ == '__main__':
    c = CrawlerProcess()
    c.crawl(StackSpider)
    c.start()

The problem is the above script gives me the same title multiple times and then another title and so on.

What possible chage should I bring about to make my script work in the right way?