rp.set_url("https://news.ycombinator.com/news/robots.txt")
rp.read()
# Reads the robots.txt
rp.can_fetch("*", 'https://news.ycombinator.com/news')
>>>> True