Autorise les moteurs sauf les bots
This commit is contained in:
parent
161c1bcb80
commit
ffd14a8734
@ -128,7 +128,17 @@ class SitemapGenerator
|
|||||||
*/
|
*/
|
||||||
private $sampleRobotsLines = [
|
private $sampleRobotsLines = [
|
||||||
"User-agent: *",
|
"User-agent: *",
|
||||||
|
"Disallow: /",
|
||||||
|
"User-agent: Googlebot",
|
||||||
"Allow: /",
|
"Allow: /",
|
||||||
|
"User-agent: bingbot",
|
||||||
|
"Allow: /",
|
||||||
|
"User-agent: Slurp",
|
||||||
|
"Allow: /",
|
||||||
|
"User-agent: DuckDuckBot",
|
||||||
|
"Allow: /",
|
||||||
|
"User-agent: Baiduspider",
|
||||||
|
"Allow: /"
|
||||||
];
|
];
|
||||||
/**
|
/**
|
||||||
* @var array list of valid changefreq values according to the spec
|
* @var array list of valid changefreq values according to the spec
|
||||||
|
Loading…
Reference in New Issue
Block a user