A question about robots.txt file

Tommy Matalino Level 1

October 22, 2012, 9:26 am

Write the reason you're deleting this FAQ

A question about robots.txt file

Do we really need to manually create the Robots.txt file?

Comments

Please login or sign up to leave a comment

Join

Beverly

It's possible, you could use a premade robots.txt file, such as a template used by another site and change any relevant details to what your site needs. That would almost be automatic but would allow you to have input.

13 years ago

Are you sure you want to delete this post?

Linksdepth

It's possible, you could use a premade robots.txt file, such as a template used by another site and change any relevant details to what your site needs. That would almost be automatic but would allow you to have input.

Can U suggest me giving some example please........

13 years ago

Are you sure you want to delete this post?

Corzhens

I guess what you mean about the robot.txt file is if you need that for the search engines to crawl on your site. From what I understand about the robot.txt file it is a guide for the search engine crawlers for the direction which parts of the web page that it should see and which parts that the crawlers should not see. But without the robot.txt file the crawler will check on everything that is on the web page which is the standard. In other words you don’t need the robot.txt file.

7 years ago

Are you sure you want to delete this post?

SEOClerks

A question about robots.txt file

A question about robots.txt file

Comments

Please login or sign up to leave a comment

Beverly

Linksdepth

Corzhens

Invalid SQL

Expensive SQL

Suspicious SQL

Match	Count	SQL	Script
SELECT * FROM `questions` as q JOIN categories_faq as c ON q.catid=c.CATID WHERE (q.status=1 OR (q.status=2 AND userid='') ) AND q.quesid='15111'	1	SELECT q.*, c.seo as CatSEO, c.name as CatName, c.parentid FROM `questions` as q JOIN categories_faq as c ON q.catid=c.CATID WHERE (q.status=1 OR (q.status=2 AND userid='') ) AND q.quesid='15111'	/opt/clerks-staging/docroot/viewfaq.php 496 () ()
SELECT * FROM seoclerks.members WHERE USERID='43'	1	SELECT * FROM seoclerks.members WHERE USERID='43'	/opt/clerks-staging/docroot/include/functions/includes/member.php 445 GetAllUserDetails() ()
SELECT * FROM seoclerks.members WHERE USERID=43	1	SELECT googleplus_profile FROM seoclerks.members WHERE USERID=43	/opt/clerks-staging/docroot/include/functions/main.php 16914 GetGooglePlusProfileFromId() ()
SELECT * FROM ratings_faq as r, seoclerks.members as m WHERE r.USERID=m.USERID AND r.upvote=1 AND r.PID=15111 LIMIT 5	1	SELECT m.username FROM ratings_faq as r, seoclerks.members as m WHERE r.USERID=m.USERID AND r.upvote=1 AND r.PID=15111 LIMIT 5	/opt/clerks-staging/docroot/include/functions/main.php 17159 GetVoters() ()
SELECT * FROM answers a, seoclerks.members b WHERE a.quesid='15111' AND a.userid=b.USERID and b.status='1' AND a.status=1 ORDER BY a.combined_votes DESC, a.date_answered asc	1	SELECT a.answer, a.USERID, a.upvotes, a.downvotes, a.ansid, a.parentid, a.combined_votes, a.date_answered, b.username, b.userlevel, b.profilepicture FROM answers a, seoclerks.members b WHERE a.quesid='15111' AND a.userid=b.USERID and b.status='1' AND a.status=1 ORDER BY a.combined_votes DESC, a.date_answered asc	/opt/clerks-staging/docroot/viewfaq.php 1245 () ()
SELECT * FROM ratings_faqanswers as r, seoclerks.members as m WHERE r.USERID=m.USERID AND r.upvote=1 AND r.PID=80101 LIMIT 5	1	SELECT m.username FROM ratings_faqanswers as r, seoclerks.members as m WHERE r.USERID=m.USERID AND r.upvote=1 AND r.PID=80101 LIMIT 5	/opt/clerks-staging/docroot/include/functions/main.php 17159 GetVoters() ()
SELECT * FROM ratings_faqanswers as r, seoclerks.members as m WHERE r.USERID=m.USERID AND r.upvote=1 AND r.PID=80102 LIMIT 5	1	SELECT m.username FROM ratings_faqanswers as r, seoclerks.members as m WHERE r.USERID=m.USERID AND r.upvote=1 AND r.PID=80102 LIMIT 5	/opt/clerks-staging/docroot/include/functions/main.php 17159 GetVoters() ()
SELECT * FROM ratings_faqanswers as r, seoclerks.members as m WHERE r.USERID=m.USERID AND r.upvote=1 AND r.PID=184829 LIMIT 5	1	SELECT m.username FROM ratings_faqanswers as r, seoclerks.members as m WHERE r.USERID=m.USERID AND r.upvote=1 AND r.PID=184829 LIMIT 5	/opt/clerks-staging/docroot/include/functions/main.php 17159 GetVoters() ()
UPDATE questions SET total_views = total_views + 1 WHERE quesid='15111'	1	UPDATE questions SET total_views = total_views + 1 WHERE quesid='15111'	/opt/clerks-staging/docroot/include/functions/main.php 1765 update_Faqviewcount() ()
SELECT * FROM questions WHERE quesid!='15111' AND status='1' AND (question like '%A question robots.txt file%' OR question like '%question%' OR question like '%robots.txt%' OR question like '%file%') -- ORDER BY RAND() LIMIT 15	1	SELECT quesid, question, seo, userid FROM questions WHERE quesid!='15111' AND status='1' AND (question like '%A question robots.txt file%' OR question like '%question%' OR question like '%robots.txt%' OR question like '%file%') -- ORDER BY RAND() LIMIT 15	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() ()
SELECT * FROM seoclerks.members WHERE USERID='337' limit 1	1	SELECT profilepicture FROM seoclerks.members WHERE USERID='337' limit 1	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() insert_get_member_profilepicture()
SELECT * FROM seoclerks.members WHERE USERID='1628' limit 1	1	SELECT profilepicture FROM seoclerks.members WHERE USERID='1628' limit 1	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() insert_get_member_profilepicture()
SELECT * FROM seoclerks.members WHERE USERID='632' limit 1	1	SELECT profilepicture FROM seoclerks.members WHERE USERID='632' limit 1	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() insert_get_member_profilepicture()
SELECT * FROM seoclerks.members WHERE USERID='3250' limit 1	1	SELECT profilepicture FROM seoclerks.members WHERE USERID='3250' limit 1	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() insert_get_member_profilepicture()
SELECT * FROM seoclerks.members WHERE USERID='9' limit 1	1	SELECT profilepicture FROM seoclerks.members WHERE USERID='9' limit 1	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() insert_get_member_profilepicture()
SELECT * FROM members_ledger WHERE script='/opt/clerks-staging/docroot/viewfaq.php' AND querystring LIKE '%id=15111%' AND added>=UNIX_TIMESTAMP(NOW())-1200 GROUP BY USERID	1	SELECT USERID, username FROM members_ledger WHERE script='/opt/clerks-staging/docroot/viewfaq.php' AND querystring LIKE '%id=15111%' AND added>=UNIX_TIMESTAMP(NOW())-1200 GROUP BY USERID	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() ()
SELECT * FROM categories	1	SELECT * FROM categories	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() parseRedundantQueriesCache()
select * from categories_software order by name asc	1	select * from categories_software order by name asc	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() insert_GetSoftwareCategories()
select * from categories_wanttobuy order by name asc	1	select * from categories_wanttobuy order by name asc	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() insert_get_wantcategories()
select * from categories_wanttotrade order by name asc	1	select * from categories_wanttotrade order by name asc	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() insert_get_tradecategories()
SELECT * FROM seoclerks.members WHERE USERID='147.jpg'	1	SELECT profilepicture FROM seoclerks.members WHERE USERID='147.jpg'	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() getUserProfileImage()
SELECT * FROM seoclerks.members WHERE USERID='600657.jpg'	1	SELECT profilepicture FROM seoclerks.members WHERE USERID='600657.jpg'	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() getUserProfileImage()
select * from categories_faq order by name asc	1	select * from categories_faq order by name asc	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() insert_GetFaqCategories()